I’ve been using java’s StringTokenizer lately and recently needed to use the same functionality in c++ for one of my classes, so I wrote a c++ utility class to handle this. It is presented much like the java tokenizer class but obviously toned down a bit.

/* @class Tokenizes a string and generates elements
 */
class Tokenizer {

private: 

	vector _elements;
	string _data;
	char _delimiter;
	int _index;

public:

	Tokenizer(string value, const char delimiter = ‘ ‘)
	: _data(value)
	, _delimiter(delimiter)
	, _index(0)
	{
		//tokenize our input so its ready to be transversed
		tokenize();
	}

	/* @brief Provides the next element to the caller
	 * @param element The string to store the element (passed by reference)
	 * @return Whether or no we have reached the end
	 */
	bool next(string& element) {

		if(_index >= _elements.size())
			return false; 

		element =  _elements[_index];
		++_index;
		return true;
	}

	/* @brief Resets the internal pointer to the beginning of the element
	 * list
	 */
	bool first() {
		_index = 0;
	}

private:

	/* @brief Seperates and stores each element in the string
	 */
	void tokenize() {

		int first = 0;
		int last = 0;

		do {
			//search for the first or next token
			last = _data.find_first_of(_delimiter, first);	

			if(last == -1) {
				if(first == _data.size())
				 	//there no tokens or characters left, exit
					break;
				else {
					//still one element left but no final token
					last = _data.size();
					_elements.push_back(_data.substr(first, last - first));
					break;
				}
			}
			else {
				//found token, store element
				_elements.push_back(_data.substr(first, last - first));
				first = last + 1;
			}

		} while(last != -1 );
	}

};