From bb381be55f3c2d8441a000a6cd3a9f8030477c3a Mon Sep 17 00:00:00 2001
From: Pedro <pedroflemming@gmail.com>
Date: Fri, 17 Oct 2014 17:43:57 +0200
Subject: [PATCH] extended flex to use custom token type

---
 lemon_example/build_and_run.sh |  2 +-
 lemon_example/flex_scanner.l   | 11 ++++++++---
 lemon_example/lemon_parser.y   | 12 ++++++------
 lemon_example/main.cpp         | 24 +++++++++++++++---------
 lemon_example/token_def.h      |  9 +++++++++
 5 files changed, 39 insertions(+), 19 deletions(-)
 create mode 100644 lemon_example/token_def.h

diff --git a/lemon_example/build_and_run.sh b/lemon_example/build_and_run.sh
index 57bcabe..32e2206 100644
--- a/lemon_example/build_and_run.sh
+++ b/lemon_example/build_and_run.sh
@@ -1,4 +1,4 @@
-rm -f lemon_test lemon_parser.c flex_scanner.c
+rm -f lemon_test lemon_parser.c lemon_parser.h flex_scanner.c flex_scanner.h
 echo "Lemon"
 lemon lemon_parser.y
 echo "Flex"
diff --git a/lemon_example/flex_scanner.l b/lemon_example/flex_scanner.l
index c47c4ff..b8e512f 100644
--- a/lemon_example/flex_scanner.l
+++ b/lemon_example/flex_scanner.l
@@ -1,15 +1,20 @@
 %{
-    #include "lemon_parser.h"
-    #include <stdlib.h>
+
+#include "lemon_parser.h"
+#include <stdlib.h>
+#include "token_def.h"
+
+
 %}
 
 
 %option reentrant
 %option noyywrap
+%option bison-bridge
 
 
 %%
-[0-9]+|[0-9]+.[0-9]+      return NUMBER;
+[0-9]+|[0-9]+.[0-9]+      { yylval->fval = atof(yytext); return NUMBER; }
 [+]         return PLUS;
 [\n]        return NL;
 [ \t]       ;       /* skip whitespace */
diff --git a/lemon_example/lemon_parser.y b/lemon_example/lemon_parser.y
index f811734..5e29119 100644
--- a/lemon_example/lemon_parser.y
+++ b/lemon_example/lemon_parser.y
@@ -1,16 +1,16 @@
 %include {
 	#include <assert.h>
 	#include <cstdlib>
+	#include "token_def.h"
 }
 %syntax_error { printf("Lemon syntax error\n"); }
 
-%token_type {const char*}
-%type expr {float}
-
-%left PLUS MINUS .
-
+%token_type {LexerToken*}
 %extra_argument { float* result }
 
+%type expr {float}
+%left PLUS MINUS .
+
 start   ::= prog.
 
 prog    ::= prog print NL .
@@ -19,5 +19,5 @@ prog    ::= .
 
 print   ::= expr(a) .         { *result = a; }
 
-expr(a) ::= NUMBER(b) .                { a = atof(b); }
+expr(a) ::= NUMBER(b) .                { a = b->fval; }
 expr(a) ::= expr(b) PLUS expr(c) .      { a = b + c; }
\ No newline at end of file
diff --git a/lemon_example/main.cpp b/lemon_example/main.cpp
index 069310f..382cadf 100644
--- a/lemon_example/main.cpp
+++ b/lemon_example/main.cpp
@@ -1,8 +1,9 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <thread>
-#include "flex_scanner.h"
+#include "token_def.h"
 #include "lemon_parser.h"
+#include "flex_scanner.h"
 
 // Based on https://github.com/theory/flex-lemon-example
 // and http://stackoverflow.com/questions/24833465/bison-yacc-vs-lemon-vs-standard-input
@@ -10,10 +11,10 @@ typedef float ResultType;
 
 void *ParseAlloc(void *(*mallocProc)(size_t));
 void ParseFree(void *p, void (*freeProc)(void*));
-void Parse(void *yyp, int yymajor, const char* text, ResultType*);
+void Parse(void *yyp, int yymajor, LexerToken* token, ResultType*);
 
-int yylex(void);
-int yylval;
+// int yylex(void);
+// int yylval;
 
 
 float parseString(const char* string) {
@@ -27,15 +28,16 @@ float parseString(const char* string) {
 	int tokenCode;
 	ResultType result;
 	do {
-		tokenCode = yylex(scanner);
-		Parse(lemonParser, tokenCode, yyget_text(scanner), &result);
+		LexerToken token;
+		tokenCode = yylex(&token, scanner);
+		Parse(lemonParser, tokenCode, &token, &result);
 		// printf("Token %d\n", tokenCode);
 	} while (tokenCode > 0);
 
 	return result;
 }
 
-
+int parse_count = 0;
 void multithreadTest(int numOfParses, int id) {
 	for (int n = 0; n < numOfParses; ++n) {
 		int a = rand() % 1000 + 1;
@@ -44,14 +46,18 @@ void multithreadTest(int numOfParses, int id) {
 		char string[32];
 		sprintf(string, "%d + %d", a, b);
 
+		parse_count++;
 		int result = parseString(string);
+		if (parse_count != 1) printf("+");
+		parse_count--;
+
 		if (result != c) printf("Error[%d]! %s != %d\n", id, string, result);
 	}
 }
 
 int main(void) {
-	const int numThreads = 10;
-	int numRuns = 5000;
+	const int numThreads = 20;
+	int numRuns = 300;
 
 	std::thread threads[numThreads];
 	for (int n = 0; n < numThreads; ++n) {
diff --git a/lemon_example/token_def.h b/lemon_example/token_def.h
new file mode 100644
index 0000000..fd13ab5
--- /dev/null
+++ b/lemon_example/token_def.h
@@ -0,0 +1,9 @@
+
+
+struct LexerToken {
+	char* sval;
+	float fval;
+};
+
+#define YYSTYPE LexerToken
+