/* Copyright Motorola, Inc. 1993, 1994 ALL RIGHTS RESERVED You are hereby granted a copyright license to use, modify, and distribute the SOFTWARE so long as this entire notice is retained without alteration in any modified and/or redistributed versions, and that such modified versions are clearly identified as such. No licenses are granted by implication, estoppel or otherwise under any patents or trademarks of Motorola, Inc. The SOFTWARE is provided on an "AS IS" basis and without warranty. To the maximum extent permitted by applicable law, MOTOROLA DISCLAIMS ALL WARRANTIES WHETHER EXPRESS OR IMPLIED, INCLUDING IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE AND ANY WARRANTY AGAINST INFRINGEMENT WITH REGARD TO THE SOFTWARE (INCLUDING ANY MODIFIED VERSIONS THEREOF) AND ANY ACCOMPANYING WRITTEN MATERIALS. To the maximum extent permitted by applicable law, IN NO EVENT SHALL MOTOROLA BE LIABLE FOR ANY DAMAGES WHATSOEVER (INCLUDING WITHOUT LIMITATION, DAMAGES FOR LOSS OF BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF BUSINESS INFORMATION, OR OTHER PECUNIARY LOSS) ARISING OF THE USE OR INABILITY TO USE THE SOFTWARE. Motorola assumes no responsibility for the maintenance and support of the SOFTWARE. */ /* these are the data structures that we need to do the prototypes for all the functions */ /* tokenizer TB header file This contains all the data structures that are needed by the toolbox. */ #include "errors.h" /* these contain the parameters for the TB that can be changed. (string sizes etc) */ #define IB_SIZE 1000 #define MAX_INPUT_BUFFERS 3 #define MAX_LEXEME_LENGTH 34 /* PAF 20 */ /* these defines are needed for this toolbox and they should not be redefined so we will put them in this toolbox header file */ /* token translation list status defines */ #define SORTED 1 #define NOT_SORTED 0 /* this is the mirrored status */ #define MIRRORED 1 #define NOT_MIRRORED 0 /* input buffer modes */ #define DIRECT 2 #define STREAM 3 /* reserved tokens */ #define UNK_TOK 0xffff #define UNDEF_TOK 0xfffe /* these are defines which represent data types that are assumed in the system.*/ /* the TB defines a token to be of this type */ #define TOKEN unsigned long /* this is a trick designed to only allow one definition for the variables and the functions in here... */ #ifdef IN_TOK_TB #define WHERE #else #define WHERE extern #endif /*--------------- Token Translation List ELEMENT ---------------------------*/ /* this is the definition for ONE token definition. THis stores the string ('\0' terminated) and the associate value for that lexeme */ typedef struct { /* here is the NULL terminated string definition */ char lexeme[MAX_LEXEME_LENGTH]; /* here is the value for that lexeme */ TOKEN value; } tok_trans_list_element; /*--------------- Token Translation List ----------------------------------*/ typedef struct { /* here is how the ttl is created... the programmer needs to declare an array of tok_trans_list_element and the tok_trans_list will point to the head of that list. This allows the programmer to only allocate the memory needed for the token list. Not preset length for the token list is necessary in this case. */ tok_trans_list_element *list; /* these are pointers to the lists of delimiter characters, symbol token characters and identifier characters respectively. Notice that these are pointers to strings in memory. If the memory that these point to are changed the values that the values that the TB uses will subsequently change */ char *delimiters; char *symbol_tokens; char *ident_chars; /* since the number of elements is "variable" the TB needs to know how many elements there are... so here is the definition to do so. */ int count; /* this is for convience... If the programmer knows that the list is presorted then he can just set this to SORTED and the TB will know that everything is ok */ char sorted; } tok_trans_list; /* --------------------- ib_element definition ------------------------------*/ typedef struct { /* this is the actual buffer that is to be allocated. Notice that the size is a fixed length and this is set by a DEFINED value */ char buffer[IB_SIZE]; /* this is the file that is associated with this input buffer. This could be STDIN or an actual file */ char *input_device; /* this structure stores all the pointers that are needed to access the buffer as the specs dictate */ struct { char *prevStart, *prevEnd; char *currentStart, *currentEnd; char *nextStart, *nextEnd; char *buffer_end; }ptrs; /* this structure holds the different states that the input buffer could possibly be in. mode is DIRECT or STREAM mirror is MIRRORED or NOT_MIRRORED */ struct { char mode; char mirror; char not_forwarded_yet; STATUS floatnext; } stat; /* these are the pointers to the routines that are needed to open, read, and close the input buffer input sources. */ struct { /* the open function is not needed because the tok_open_input_buffer will use it the ONE time that it needs to be used */ /* STATUS (*openfn)(); */ STATUS (*readfn)(); STATUS (*closefn)(); } funcs; } ib_element; /*------------------------------------------------------------------------------*/ /* this is the definition of the information that needs to be kept for the whole toolbox in general. ::::::::::::::: NOTICE :::::::::::::: This structure is defined to be unique for any system so ONLY ONE copy can exist in memory at any one time. That is why the definition is as follows!!!!!! ::::::::::::::: NOTICE II ::::::::::: Yes... You are right this is a nested structure definition. This is to promote understanding and conversion to C++ in the future. Don't mess with it now. ttl stands for Token Translation List and ib stands for Input Buffer. */ WHERE struct { struct { /* this is a pointer to the current token translation list */ tok_trans_list *current; } ttl; struct { /* the number of Input Buffers Currently open (this is the same as the current input buffer index number */ int current; /* this is the actual Input Buffer Array. this is where all the input buffers are physically stored. */ ib_element list[MAX_INPUT_BUFFERS]; } ib; } tok_toolbox_data; /*----------------------------------------------------------------------- these will help you get to the data within the tok_toolbox_data structure easier. !!!!!!!!!!!!!!!!!! SINCE THIS IS A CLOSED TOOLBOX THESE DEFINES SHOULD ONLY BE USED BY THE TOOLBOX ITSELF !!!!!!!!!!!!!!!!! */ /* this gets you to the input buffer section of the toolbox data*/ #define TOK_IB tok_toolbox_data.ib /* this gets you the current input buffer number */ #define TOK_CIBN TOK_IB.current /* this gets you to the current input buffer data structure so that you can access the information in there. */ #define TOK_CIBE TOK_IB.list[TOK_CIBN] /* this gets you a pointer to the current input buffer's character buffer */ #define TOK_CIBEB TOK_CIBE.buffer /* this gets you to the pointers in the current input buffer */ #define TOK_CIBEP TOK_CIBE.ptrs /* this gets you to the status in the current input buffer */ #define TOK_CIBES TOK_CIBE.stat /* this will make it easier to get to the functions of the IB */ #define TOK_CIBEF TOK_CIBE.funcs /* this gets you to the token translation list section of the toolbox data */ #define TOK_TTL tok_toolbox_data.ttl /* this gets you a pointer to the current Token Translation List */ #define TOK_CTTL TOK_TTL.current /* this allows this to be used as an include file in any other files without redefining the WHERE define */ WHERE STATUS tok_initialize_toolbox(); WHERE STATUS tok_reset(); WHERE STATUS tok_set_delimiters(/*char **/); WHERE STATUS tok_set_symbol_tokens(); WHERE STATUS tok_set_identifier_characters(/*char **/); WHERE STATUS tok_set_translation_list(/*tok_trans_list **/); WHERE STATUS tok_mode_direct_input(); WHERE STATUS tok_mode_stream_input(); WHERE STATUS tok_sort_translation_list(); WHERE STATUS tok_pull_out_string(/*char *,char *,char **/); WHERE STATUS tok_string_to_token(/*TOKEN *,char *,char **/); WHERE STATUS tok_find_next_delimited_string(/*char **,char **,char **/); WHERE STATUS tok_is_next_token(/*TOKEN*/); WHERE STATUS tok_get_next_token(/*TOKEN **/); WHERE STATUS tok_get_token_string(/*char **/); WHERE STATUS tok_move_forward(); WHERE STATUS tok_move_backward(); WHERE STATUS tok_flush_buffer(); WHERE STATUS tok_fill_buffer(); #undef WHERE