From 8d037836c8e9855531385e9c74958dda71964bcb Mon Sep 17 00:00:00 2001
From: Jan Breuer <jan.breuer@jaybee.cz>
Date: 摹曛, 21 3月 2013 23:44:56 +0800
Subject: [PATCH] Complete SCPI Lexer and unit test

---
 libscpi/src/lexer.c       |  324 +++++++++++++++-------------
 libscpi/Makefile          |    5 
 libscpi/test/test_lexer.c |  190 +++++++++++++++++
 libscpi/inc/scpi/lexer.h  |  104 +++++++++
 4 files changed, 471 insertions(+), 152 deletions(-)

diff --git a/libscpi/Makefile b/libscpi/Makefile
index d20e0a4..e58cd39 100644
--- a/libscpi/Makefile
+++ b/libscpi/Makefile
@@ -23,6 +23,7 @@
 SRCS = $(addprefix src/, \
 	debug.c error.c fifo.c ieee488.c \
 	minimal.c parser.c units.c utils.c \
+	lexer.c \
 	)
 
 OBJS = $(addprefix $(OBJDIR)/, $(notdir $(SRCS:.c=.o)))
@@ -30,12 +31,12 @@
 HDRS = $(addprefix inc/scpi/, \
 	scpi.h constants.h debug.h error.h \
 	fifo.h ieee488.h minimal.h parser.h \
-	types.h units.h \
+	types.h units.h lexer.h \
 	) src/utils.h
 
 
 TESTS = $(addprefix test/, \
-	test_fifo.c test_scpi_utils.c \
+	test_fifo.c test_scpi_utils.c test_lexer.c \
 	)
 
 TESTS_OBJS = $(TESTS:.c=.o)
diff --git a/libscpi/inc/scpi/lexer.h b/libscpi/inc/scpi/lexer.h
new file mode 100644
index 0000000..c2124b9
--- /dev/null
+++ b/libscpi/inc/scpi/lexer.h
@@ -0,0 +1,104 @@
+/*-
+ * Copyright (c) 2012-2013 Jan Breuer,
+ *
+ * All Rights Reserved
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * @file   lexer.h
+ * @date   Thu Mar 21 15:00:58 UTC 2013
+ * 
+ * @brief  SCPI Lexer
+ * 
+ * 
+ */
+
+#ifndef SCPI_LEXER_H
+#define	SCPI_LEXER_H
+
+#include "scpi/types.h"
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+enum _token_type_t {
+    TokComma,
+    TokSemicolon,
+    TokQuiestion,
+    TokNewLine,
+    TokHexnum,
+    TokOctnum,
+    TokBinnum,
+    TokProgramMnemonic,
+    TokDecimalNumericProgramData,
+    TokSuffixProgramData,
+    TokArbitraryBlockProgramData,
+    TokSingleQuoteProgramData,
+    TokDoubleQuoteProgramData,
+    TokProgramExpression,
+    TokCompoundProgramHeader,
+    TokCommonProgramHeader,
+    TokWhiteSpace,
+    TokUnknown,
+};
+typedef enum _token_type_t token_type_t;
+
+struct _token_t {
+    token_type_t type;
+    const char * ptr;
+    int len;
+};
+typedef struct _token_t token_t;
+
+struct _lex_state_t {
+    const char * buffer;
+    const char * pos;
+    int len;
+};
+typedef struct _lex_state_t lex_state_t;
+
+
+int SCPI_LexWhiteSpace(lex_state_t * state, token_t * token);
+int SCPI_LexCommonProgramHeader(lex_state_t * state, token_t * token);
+int SCPI_LexCompoundProgramHeader(lex_state_t * state,  token_t * token);
+int SCPI_LexProgramHeader(lex_state_t * state,  token_t * token);
+int SCPI_LexQuestion(lex_state_t * state, token_t * token);
+int SCPI_LexCharacterProgramData(lex_state_t * state, token_t * token);
+int SCPI_LexDecimalNumericProgramData(lex_state_t * state, token_t * token);
+int SCPI_LexSuffixProgramData(lex_state_t * state, token_t * token);
+int SCPI_LexNondecimalNumericData(lex_state_t * state, token_t * token);
+int SCPI_LexStringProgramData(lex_state_t * state,  token_t * token);
+int SCPI_LexArbitraryBlockProgramData(lex_state_t * state, token_t * token);
+int SCPI_LexProgramExpression(lex_state_t * state,  token_t * token);
+int SCPI_LexComma(lex_state_t * state, token_t * token);
+int SCPI_LexSemicolon(lex_state_t * state, token_t * token);
+int SCPI_LexNewLine(lex_state_t * state,  token_t * token);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* SCPI_LEXER_H */
+
diff --git a/libscpi/src/lexer.c b/libscpi/src/lexer.c
index f145ca0..abca2de 100644
--- a/libscpi/src/lexer.c
+++ b/libscpi/src/lexer.c
@@ -38,40 +38,8 @@
 #include <stdio.h>
 #include <string.h>
 
-enum _token_type_t {
-    TokComma,
-    TokSemicolon,
-    TokQuiestion,
-    TokNewLine,
-    TokHexnum,
-    TokOctnum,
-    TokBinnum,
-    TokProgramMnemonic,
-    TokDecimalNumericProgramData,
-    TokSuffixProgramData,
-    TokSingleQuoteProgramData,
-    TokDoubleQuoteProgramData,
-    TokProgramExpression,
-    TokCompoundProgramHeader,
-    TokCommonProgramHeader,
-    TokWhiteSpace,
-    TokUnknown,
-};
-typedef enum _token_type_t token_type_t;
+#include "scpi/lexer.h"
 
-struct _token_t {
-    token_type_t type;
-    const char * ptr;
-    int len;
-};
-typedef struct _token_t token_t;
-
-struct _lex_state_t {
-    const char * buffer;
-    const char * pos;
-    int len;
-};
-typedef struct _lex_state_t lex_state_t;
 
 /* identify character */
 static int isws ( int c ) {
@@ -357,7 +325,7 @@
         someNumbers += skipNumbers(state);
     }
     
-    return someNumbers > 0;
+    return someNumbers;
 }
 
 static int skipExponent(lex_state_t * state) {
@@ -370,10 +338,10 @@
         
         skipPlusmn(state);
         
-        someNumbers += skipNumbers(state);
+        someNumbers = skipNumbers(state);
     }
     
-    return someNumbers > 0;
+    return someNumbers;
 }
 
 int SCPI_LexDecimalNumericProgramData(lex_state_t * state, token_t * token) {
@@ -386,6 +354,8 @@
         if(!skipExponent(state)) {
             state->pos = rollback;
         }
+    } else {
+        state->pos = token->ptr;
     }
     
     token->len = state->pos - token->ptr;
@@ -490,15 +460,178 @@
 
 
 /* 7.7.5 <STRING PROGRAM DATA> */
-// TODO: int SCPI_LexStringProgramData(lex_state_t * state,  token_t * token)
-// void SCPI_LexSingleQuoteProgramData(lex_state_t * state)
-// void SCPI_LexDoubleQuoteProgramDatalex_state_t * state)
+static int isascii7bit(int c) {
+    return (c >= 0) && (c <= 0x7f);
+}
+
+
+static int skipQuoteProgramData(lex_state_t * state, int quote) {
+    while(!iseos(state)) {
+        if (isascii7bit(state->pos[0]) && !ischr(state, quote)) {
+            state->pos++;
+        } else if (ischr(state, quote)) {
+            state->pos++;
+            if (!iseos(state) && ischr(state, quote)) {
+                state->pos++;
+            } else {
+                state->pos--;
+                break;
+            }
+        }
+    }
+}
+    
+static int skipDoubleQuoteProgramData(lex_state_t * state) {
+    skipQuoteProgramData(state, '"');
+}
+
+static int skipSingleQuoteProgramData(lex_state_t * state) {
+    skipQuoteProgramData(state, '\'');
+}
+
+int SCPI_LexStringProgramData(lex_state_t * state,  token_t * token) {
+    token->ptr = state->pos;
+    
+    if (!iseos(state)) {
+        if (ischr(state, '"')) {
+            state->pos++;
+            token->type = TokDoubleQuoteProgramData;
+            skipDoubleQuoteProgramData(state);
+            if (!iseos(state) && ischr(state, '"')) {
+                state->pos++;
+                token->len = state->pos - token->ptr;
+            } else {
+                state->pos = token->ptr;
+            }
+        } else if (ischr(state, '\'')) {
+            state->pos++;
+            token->type = TokSingleQuoteProgramData;
+            skipSingleQuoteProgramData(state);
+            if (!iseos(state) && ischr(state, '\'')) {
+                state->pos++;
+                token->len = state->pos - token->ptr;
+            } else {
+                state->pos = token->ptr;
+            }
+        }
+    }
+    
+    token->len = state->pos - token->ptr;
+    
+    if((token->len > 0)) {
+        token->ptr++;
+        token->len -= 2;
+    } else {
+        token->type = TokUnknown;
+        state->pos = token->ptr;
+        token->len = 0;
+    }
+    
+    return token->len;
+}
 
 /* 7.7.6 <ARBITRARY BLOCK PROGRAM DATA> */
-// TODO: int SCPI_LexArbitraryBlockProgramData(lex_state_t * state,  token_t * token)
+static int isNonzeroDigit(int c) {
+    return isdigit(c) && (c != '0');
+}
+
+int SCPI_LexArbitraryBlockProgramData(lex_state_t * state, token_t * token) {
+    int i;
+    int j = 0;
+    
+    token->ptr = state->pos;
+
+    if (skipChr(state, '#')) {
+        if (!iseos(state) && isNonzeroDigit(state->pos[0])) {
+            /* Get number of digits */
+            i = state->pos[0] - '0';
+            state->pos++;
+            
+            for(; i > 0; i--) {
+                if (!iseos(state) && isdigit(state->pos[0])) {
+                    j *= 10;
+                    j += (state->pos[0] - '0');
+                    state->pos++;
+                } else {
+                    break;
+                }
+            }
+            
+            if(i == 0) {
+                state->pos += j;
+                if (!iseos(state)) {
+                    token->ptr = state->pos - j;
+                    token->len = j;
+                } else {
+                    token->len = 0;
+                }
+            } else {
+                token->len = 0;
+            }
+        } else {
+            token->len = 0;
+        }
+    }
+
+    if((token->len > 0)) {
+        token->type = TokArbitraryBlockProgramData;
+    } else {
+        token->type = TokUnknown;
+        state->pos = token->ptr;
+        token->len = 0;
+    }
+    
+    return token->len;
+}
 
 /* 7.7.7 <EXPRESSION PROGRAM DATA> */
-// TODO: int SCPI_LexProgramExpression(lex_state_t * state,  token_t * token)
+static int isProgramExpression(int c) {
+    if ((c >= 0x20) && (c<=0x7e)) {
+        if ((c!= 0x22)
+            && (c!= 0x23) 
+            && (c!= 0x27) 
+            && (c!= 0x28) 
+            && (c!= 0x29) 
+            && (c!= 0x3B)) {
+            return 1;
+        }
+    }
+    
+    return 0;
+}
+
+static void skipProgramExpression(lex_state_t * state) {
+    while(!iseos(state) && isProgramExpression(state->pos[0])) {
+        state->pos++;
+    }
+}
+
+// TODO: 7.7.7.2-2 recursive - any program data
+int SCPI_LexProgramExpression(lex_state_t * state,  token_t * token) {
+    token->ptr = state->pos;
+    
+    if (!iseos(state) && ischr(state, '(')) {
+        state->pos++;
+        skipProgramExpression(state);
+        
+        if (!iseos(state) && ischr(state, ')')) {
+            state->pos++;
+            token->len = state->pos - token->ptr;
+        } else {
+            token->len = 0;
+        }
+    }
+    
+    if((token->len > 0)) {
+        token->type = TokProgramExpression;
+    } else {
+        token->type = TokUnknown;
+        state->pos = token->ptr;
+        token->len = 0;
+    }
+    
+    return token->len;
+}
 
 int SCPI_LexComma(lex_state_t * state, token_t * token) {
     token->ptr = state->pos;
@@ -547,114 +680,5 @@
     return token->len;
 }
 
-const char * typeToStr(token_type_t type) {
-    switch(type) {
-        case TokComma: return "TokComma";
-        case TokSemicolon: return "TokSemicolon";
-        case TokQuiestion: return "TokQuiestion";
-        case TokNewLine: return "TokNewLine";
-        case TokHexnum: return "TokHexnum";
-        case TokOctnum: return "TokOctnum";
-        case TokBinnum: return "TokBinnum";
-        case TokProgramMnemonic: return "TokProgramMnemonic";
-        case TokDecimalNumericProgramData: return "TokDecimalNumericProgramData";
-        case TokSuffixProgramData: return "TokSuffixProgramData";
-        case TokSingleQuoteProgramData: return "TokSingleQuoteProgramData";
-        case TokDoubleQuoteProgramData: return "TokDoubleQuoteProgramData";
-        case TokProgramExpression: return "TokProgramExpression";
-        case TokCompoundProgramHeader: return "TokCompoundProgramHeader";
-        case TokCommonProgramHeader: return "TokCommonProgramHeader";
-        case TokWhiteSpace: return "TokWhiteSpace";
-        default: return "TokUnknown";
-    }
-}
 
-void printToken(token_t * token) {
-    printf("Token:\r\n");
-    printf("\t->type = %s\r\n", typeToStr(token->type));
-    printf("\t->ptr = %p (\"%.*s\")\r\n", token->ptr, token->len, token->ptr);
-    printf("\t->len = %d\r\n", token->len);
-}
-
-#define INIT_STATE(str) do {            \
-    state.buffer = state.pos = (str);   \
-    state.len = strlen((str));          \
-} while(0)
-
-int main(int argc, char ** argv) {
-    lex_state_t state;
-    token_t token;
-    
-    
-    //    INIT_STATE("MEAS:VOLT:DC? 1, 5\r\n");
-    INIT_STATE("  \t MEAS:VOLT:DC? 1.58, .125,  5V\r\n");  
-    SCPI_LexWhiteSpace(&state, &token); printToken(&token);
-    
-    INIT_STATE("#H123fe5A , ");
-    SCPI_LexNondecimalNumericData(&state, &token); printToken(&token);
-    
-    INIT_STATE("#B0111010101 , ");
-    SCPI_LexNondecimalNumericData(&state, &token); printToken(&token);
-    
-    INIT_STATE("#Q125725433 , ");
-    SCPI_LexNondecimalNumericData(&state, &token); printToken(&token);
-    
-    INIT_STATE("abc_213as564 , ");
-    SCPI_LexCharacterProgramData(&state, &token); printToken(&token);
-    
-    INIT_STATE("10 , ");
-    SCPI_LexDecimalNumericProgramData(&state, &token); printToken(&token);
-    
-    INIT_STATE("-10.5 , ");
-    SCPI_LexDecimalNumericProgramData(&state, &token); printToken(&token);
-    
-    INIT_STATE("+.5 , ");
-    SCPI_LexDecimalNumericProgramData(&state, &token); printToken(&token);
-    
-    INIT_STATE("-. , ");
-    SCPI_LexDecimalNumericProgramData(&state, &token); printToken(&token);
-    
-    INIT_STATE("-1 e , ");
-    SCPI_LexDecimalNumericProgramData(&state, &token); printToken(&token);
-    
-    INIT_STATE("-1 e 3, ");
-    SCPI_LexDecimalNumericProgramData(&state, &token); printToken(&token);
-    
-    INIT_STATE("1.5E12 , ");
-    SCPI_LexDecimalNumericProgramData(&state, &token); printToken(&token);
-    
-    INIT_STATE("A/V , ");
-    SCPI_LexSuffixProgramData(&state, &token); printToken(&token);
-    
-    INIT_STATE("mA.h , ");
-    SCPI_LexSuffixProgramData(&state, &token); printToken(&token);
-    
-    INIT_STATE("*IDN?, ");
-    SCPI_LexCommonProgramHeader(&state, &token); printToken(&token);
-    
-    INIT_STATE("*?, ");
-    SCPI_LexCommonProgramHeader(&state, &token); printToken(&token);
-    
-    INIT_STATE("MEAS:VOLT:DC?, ");
-    SCPI_LexCommonProgramHeader(&state, &token); printToken(&token);
-    
-    INIT_STATE("MEAS:VOLT:DC?, ");
-    SCPI_LexCompoundProgramHeader(&state, &token); printToken(&token);
-    
-    INIT_STATE(":MEAS:VOLT:DC?, ");
-    SCPI_LexCompoundProgramHeader(&state, &token); printToken(&token);
-    
-    INIT_STATE(":MEAS::VOLT:DC?, ");
-    SCPI_LexCompoundProgramHeader(&state, &token); printToken(&token);
-    
-    INIT_STATE(":MEAS::VOLT:DC?, ");
-    SCPI_LexProgramHeader(&state, &token); printToken(&token);
-    
-    INIT_STATE("MEAS:VOLT:DC?, ");
-    SCPI_LexProgramHeader(&state, &token); printToken(&token);
-    
-    INIT_STATE("*IDN?, ");
-    SCPI_LexProgramHeader(&state, &token); printToken(&token);
-    return 0;
-}
 
diff --git a/libscpi/test/test_lexer.c b/libscpi/test/test_lexer.c
new file mode 100644
index 0000000..47a8dfe
--- /dev/null
+++ b/libscpi/test/test_lexer.c
@@ -0,0 +1,190 @@
+/*
+ * File:   test_lexer.c
+ * Author: jaybee
+ *
+ * Created on Thu Mar 21 14:39:03 UTC 2013
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "CUnit/Basic.h"
+
+#include "scpi/lexer.h"
+
+/*
+ * CUnit Test Suite
+ */
+
+int init_suite(void) {
+    return 0;
+}
+
+int clean_suite(void) {
+    return 0;
+}
+
+typedef int (*lexfn_t)(lex_state_t * state, token_t * token);
+
+
+const char * typeToStr(token_type_t type) {
+    switch(type) {
+        case TokComma: return "TokComma";
+        case TokSemicolon: return "TokSemicolon";
+        case TokQuiestion: return "TokQuiestion";
+        case TokNewLine: return "TokNewLine";
+        case TokHexnum: return "TokHexnum";
+        case TokOctnum: return "TokOctnum";
+        case TokBinnum: return "TokBinnum";
+        case TokProgramMnemonic: return "TokProgramMnemonic";
+        case TokDecimalNumericProgramData: return "TokDecimalNumericProgramData";
+        case TokSuffixProgramData: return "TokSuffixProgramData";
+        case TokArbitraryBlockProgramData: return "TokArbitraryBlockProgramData";
+        case TokSingleQuoteProgramData: return "TokSingleQuoteProgramData";
+        case TokDoubleQuoteProgramData: return "TokDoubleQuoteProgramData";
+        case TokProgramExpression: return "TokProgramExpression";
+        case TokCompoundProgramHeader: return "TokCompoundProgramHeader";
+        case TokCommonProgramHeader: return "TokCommonProgramHeader";
+        case TokWhiteSpace: return "TokWhiteSpace";
+        default: return "TokUnknown";
+    }
+}
+
+void printToken(token_t * token) {
+    printf("Token:\r\n");
+    printf("\t->type = %s\r\n", typeToStr(token->type));
+    printf("\t->ptr = %p (\"%.*s\")\r\n", token->ptr, token->len, token->ptr);
+    printf("\t->len = %d\r\n", token->len);
+}
+
+
+#if 0
+static void TEST_TOKEN(const char * str, lexfn_t fn, int offset, int len, token_type_t tp)  {
+    lex_state_t state;
+    token_t token;
+    
+    state.buffer = state.pos = str;
+    state.len = strlen(str);
+    fn(&state, &token);
+    CU_ASSERT_EQUAL(str + offset, token.ptr);
+    CU_ASSERT_EQUAL(len, token.len);
+    CU_ASSERT_EQUAL(tp, token.type);
+}
+#endif
+
+#define TEST_TOKEN(s, f, o, l, t) do {          \
+    const char * str = s;                       \
+    lexfn_t fn = f;                             \
+    int offset = o;                             \
+    int len = l;                                \
+    token_type_t tp = t;                        \
+    lex_state_t state;                          \
+    token_t token;                              \
+                                                \
+    state.buffer = state.pos = str;             \
+    state.len = strlen(str);                    \
+    fn(&state, &token);                         \
+    CU_ASSERT_EQUAL(str + offset, token.ptr);   \
+    CU_ASSERT_EQUAL(len, token.len);            \
+    CU_ASSERT_EQUAL(tp, token.type);            \
+    /* printToken(&token); */                   \
+} while(0)
+
+
+void testWhiteSpace(void) {
+    TEST_TOKEN("  \t MEA", SCPI_LexWhiteSpace, 0, 4, TokWhiteSpace);
+}
+
+void testNondecimal(void) {
+    TEST_TOKEN("#H123fe5A", SCPI_LexNondecimalNumericData, 2, 7, TokHexnum);
+    TEST_TOKEN("#B0111010101", SCPI_LexNondecimalNumericData, 2, 10, TokBinnum);
+    TEST_TOKEN("#Q125725433", SCPI_LexNondecimalNumericData, 2, 9, TokOctnum);
+}
+
+void testCharacterProgramData(void) {
+    TEST_TOKEN("abc_213as564 , ", SCPI_LexCharacterProgramData, 0, 12, TokProgramMnemonic);
+}
+
+void testDecimal(void) {
+    TEST_TOKEN("10 , ", SCPI_LexDecimalNumericProgramData, 0, 2, TokDecimalNumericProgramData);
+    TEST_TOKEN("-10.5 , ", SCPI_LexDecimalNumericProgramData, 0, 5, TokDecimalNumericProgramData);
+    TEST_TOKEN("+.5 , ", SCPI_LexDecimalNumericProgramData, 0, 3, TokDecimalNumericProgramData);
+    TEST_TOKEN("-. , ", SCPI_LexDecimalNumericProgramData, 0, 0, TokUnknown);
+    TEST_TOKEN("-1 e , ", SCPI_LexDecimalNumericProgramData, 0, 2, TokDecimalNumericProgramData);
+    TEST_TOKEN("-1 e 3, ", SCPI_LexDecimalNumericProgramData, 0, 6, TokDecimalNumericProgramData);
+    TEST_TOKEN("1.5E12", SCPI_LexDecimalNumericProgramData, 0, 6, TokDecimalNumericProgramData);
+}
+
+void testSuffix(void) {
+    TEST_TOKEN("A/V , ", SCPI_LexSuffixProgramData, 0, 3, TokSuffixProgramData);
+    TEST_TOKEN("mA.h", SCPI_LexSuffixProgramData, 0, 4, TokSuffixProgramData);
+}
+
+void testProgramHeader(void) {
+    TEST_TOKEN("*IDN? ", SCPI_LexCommonProgramHeader, 0, 4, TokCommonProgramHeader);
+    TEST_TOKEN("*?; ", SCPI_LexCommonProgramHeader, 0, 0, TokUnknown);
+    TEST_TOKEN("MEAS:VOLT:DC? ", SCPI_LexCommonProgramHeader, 0, 0, TokUnknown);
+    TEST_TOKEN("MEAS:VOLT:DC? ", SCPI_LexCompoundProgramHeader, 0, 12, TokCompoundProgramHeader);
+    TEST_TOKEN(":MEAS:VOLT:DC? ", SCPI_LexCompoundProgramHeader, 0, 13, TokCompoundProgramHeader);
+    TEST_TOKEN(":MEAS::VOLT:DC? ", SCPI_LexCompoundProgramHeader, 0, 6, TokCompoundProgramHeader);
+    TEST_TOKEN(":MEAS::VOLT:DC? ", SCPI_LexProgramHeader, 0, 6, TokCompoundProgramHeader);
+    TEST_TOKEN("*IDN?", SCPI_LexProgramHeader, 0, 4, TokCommonProgramHeader);
+}
+
+void testArbitraryBlock(void) {
+    TEST_TOKEN("#12AB, ", SCPI_LexArbitraryBlockProgramData, 3, 2, TokArbitraryBlockProgramData);
+    TEST_TOKEN("#13AB", SCPI_LexArbitraryBlockProgramData, 0, 0, TokUnknown);
+    TEST_TOKEN("#12\r\n, ", SCPI_LexArbitraryBlockProgramData, 3, 2, TokArbitraryBlockProgramData);
+    TEST_TOKEN("#02AB, ", SCPI_LexArbitraryBlockProgramData, 0, 0, TokUnknown);
+}
+
+void testExpression(void) {
+    TEST_TOKEN("( 1 + 2 ) , ", SCPI_LexProgramExpression, 0, 9, TokProgramExpression);
+    TEST_TOKEN("( 1 + 2  , ", SCPI_LexProgramExpression, 0, 0, TokUnknown);
+}
+
+void testString(void) {
+    TEST_TOKEN("\"ahoj\" ", SCPI_LexStringProgramData, 1, 4, TokDoubleQuoteProgramData);
+    TEST_TOKEN("'ahoj' ", SCPI_LexStringProgramData, 1, 4, TokSingleQuoteProgramData);
+    TEST_TOKEN("'ahoj ", SCPI_LexStringProgramData, 0, 0, TokUnknown);
+    TEST_TOKEN("'ah''oj' ", SCPI_LexStringProgramData, 1, 6, TokSingleQuoteProgramData);
+    TEST_TOKEN("'ah\"oj' ", SCPI_LexStringProgramData, 1, 5, TokSingleQuoteProgramData);
+    TEST_TOKEN("\"ah\"\"oj\" ", SCPI_LexStringProgramData, 1, 6, TokDoubleQuoteProgramData);
+}
+
+
+int main() {
+    CU_pSuite pSuite = NULL;
+    
+    /* Initialize the CUnit test registry */
+    if (CUE_SUCCESS != CU_initialize_registry())
+        return CU_get_error();
+    
+    /* Add a suite to the registry */
+    pSuite = CU_add_suite("Lexer", init_suite, clean_suite);
+    if (NULL == pSuite) {
+        CU_cleanup_registry();
+        return CU_get_error();
+    }
+    
+    /* Add the tests to the suite */
+    if ((NULL == CU_add_test(pSuite, "WhiteSpace", testWhiteSpace))
+        || (NULL == CU_add_test(pSuite, "Nondecimal", testNondecimal))
+        || (NULL == CU_add_test(pSuite, "CharacterProgramData", testCharacterProgramData))
+        || (NULL == CU_add_test(pSuite, "Decimal", testDecimal))
+        || (NULL == CU_add_test(pSuite, "Suffix", testSuffix))
+        || (NULL == CU_add_test(pSuite, "ProgramHeader", testProgramHeader))
+        || (NULL == CU_add_test(pSuite, "ArbitraryBlock", testArbitraryBlock))
+        || (NULL == CU_add_test(pSuite, "Expression", testExpression))
+        || (NULL == CU_add_test(pSuite, "String", testString))
+    ) {
+        CU_cleanup_registry();
+        return CU_get_error();
+    }
+    
+    /* Run all tests using the CUnit Basic interface */
+    CU_basic_set_mode(CU_BRM_VERBOSE);
+    CU_basic_run_tests();
+    CU_cleanup_registry();
+    return CU_get_error();
+}
+

--
Gitblit v1.9.1