How to make the tokenizer detect empty spaces while using strtok()
Posted
by
Shadi Al Mahallawy
on Stack Overflow
See other posts from Stack Overflow
or by Shadi Al Mahallawy
Published on 2010-12-25T18:33:46Z
Indexed on
2010/12/25
18:53 UTC
Read the original article
Hit count: 385
I am designing a c++ program, somewhere in the program i need to detect if there is a blank(empty token) next to the token used know eg.
if(token1==start)
{
token2=strtok(NULL," ");
if(token2==NULL)
{LCCTR=0;}
else
{LCCTR=atoi(token2);}
so in the previous peice token1 is pointing to start , and i want to check if there is anumber next to the start , so I used token2=strtok(NULL," ") to point to the next token but unfortunattly the strtok function cannot detect empty spaces so it gives me an error at run time"INVALID NULL POINTER" how can i fix it or is there another function to use to detect empty spaces
#include <iostream>
#include<string>
#include<map>
#include<iomanip>
#include<fstream>
#include<ctype.h>
using namespace std;
const int MAX=300;
int LCCTR;
int START(char* token1);
char* PASS1(char*token1);
void tokinizer()
{
ifstream in;
ofstream out;
char oneline[MAX];
in.open("infile.txt");
out.open("outfile.txt");
if(in.is_open())
{
char *token1;
in.getline(oneline,MAX);
token1 = strtok(oneline," \t");
START (token1);
//cout<<'\t';
while(token1!=NULL)
{
//PASS1(token1);
//cout<<token1<<" ";
token1=strtok(NULL," \t");
if(NULL==token1)
{//cout<<endl;
//cout<<LCCTR<<'\t';
in.getline(oneline,MAX);
token1 = strtok(oneline," \t");
}
}
}
in.close();
out.close();
}
int START(char* token1)
{
string start("START");
char*token2;
if(token1 != start)
{LCCTR=0;}
else if(token1==start)
{
token2=strchr(token1+2,' ');
cout<<token2;
if(token2==NULL)
{LCCTR=0;}
else
{LCCTR=atoi(token2);
if(atoi(token2)>9999||atoi(token2)<0){cout<<"IVALID STARTING ADDRESS"<<endl;exit(1);}
}
}
return LCCTR;
}
char* PASS1 (char*token1)
{
map<string,int> operations;
map<string,int>symtable;
map<string,int>::iterator it;
pair<map<string,int>::iterator,bool> ret;
char*token3=NULL;
char*token2=NULL;
string test;
string comp(" ");
string start("START");
string word("WORD");
string byte("BYTE");
string resb("RESB");
string resw("RESW");
string end("END");
operations["ADD"] = 18;
operations["AND"] = 40;
operations["COMP"] = 28;
operations["DIV"] = 24;
operations["J"] = 0X3c;
operations["JEQ"] =30;
operations["JGT"] =34;
operations["JLT"] =38;
operations["JSUB"] =48;
operations["LDA"] =00;
operations["LDCH"] =50;
operations["LDL"] =55;
operations["LDX"] =04;
operations["MUL"] =20;
operations["OR"] =44;
operations["RD"] =0xd8;
operations["RSUB"] =0x4c;
operations["STA"] =0x0c;
operations["STCH"] =54;
operations["STL"] =14;
operations["STSW"] =0xe8;
operations["STX"] =10;
operations["SUB"] =0x1c;
operations["TD"] =0xe0;
operations["TIX"] =0x2c;
operations["WD"] =0xdc;
if(operations.find("ADD")->first==token1)
{ token2=strtok(NULL," ");
//test=token2;
cout<<token2;
//if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
//else{LCCTR=LCCTR+3;}
}
/*else if(operations.find("AND")->first==token1)
{ token2=strtok(NULL," ");
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("COMP")->first==token1)
{ token2=token1+5;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("DIV")->first==token1)
{ token2=token1+4;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("J")->first==token1)
{ token2=token1+2;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("JEQ")->first==token1)
{ token2=token1+5;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("JGT")->first==token1)
{ token2=strtok(NULL," ");
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("JLT")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("JSUB")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("LDA")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("LDCH")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("LDL")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("LDX")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("MUL")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("OR")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("RD")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("RSUB")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("STA")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("STCH")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("STL")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("STSW")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("STX")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("SUB")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("TD")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("TIX")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}
else if(operations.find("WD")->first==token1)
{ token2=token1+6;
test=token2;
if(test.empty()){cout<<"MISSING OPERAND"<<endl;exit(1);}
else{LCCTR=LCCTR+3;}
}*/
//else if(
if(word==token1)
{LCCTR=LCCTR+3;}
else if(byte==token1)
{string test;
token2=token1+7;
test=token2;
if(test[0]=='C')
{token3=token1+10;
test=token3;
if(test.length()>15)
{cout<<"ERROR"<<endl;
exit(1);}
}
else if(test[0]=='X')
{token3=token1+10;
test=token3;
if(test.length()>14)
{cout<<"ERROR"<<endl;
exit(1);}
}
LCCTR=LCCTR+test.length();
}
else if(resb==token1)
{token3=token1+5;
LCCTR=LCCTR+atoi(token3);}
else if(resw==token1)
{token3=token1+5;
LCCTR=LCCTR+3*atoi(token3);}
else if(end==token1)
{exit(1);}
/*else
{
test=token1;
int last=test.length();
if(token1==start||test[0]=='C'||test[0]=='X'||ispunct(test[last])||isdigit(test[0])||isdigit(test[1])||isdigit(test[2])||isdigit(test[3])){}
else {
token2=strtok(NULL," ");
//test=token2;
cout<<token2;
if(token2!=NULL)
{
symtable.insert( pair<string,int>(token1,LCCTR));
for(it=symtable.begin() ;it!=symtable.end() ;++it)
{/*cout<<"symbol: "<<it->first<<" LCCTR: "<<it->second<<endl;}
}
else{}
}
}*/
return token3;
}
int main()
{
tokinizer();
return 0;
}
© Stack Overflow or respective owner