1 
  2 define(['jscc', 'constants', 'configurationManager', 'grammarConverter', 'jquery', 'logger', 'module'],
  3 /**
  4  * Generator for executable language-grammars (i.e. converted JSON grammars).
  5  * 
  6  * <p>
  7  * This generator uses JS/CC for compiling the JSON grammar.
  8  * 
  9  * <p>
 10  * The generator for compiling the JSON grammar definitions in <code>www/config/languages/<language code>/grammar.json</code>
 11  * can be configured in <code>www/config/configuration.json</code>:<br>
 12  * <pre>
 13  * {
 14  *   ...
 15  *   "grammarCompiler": "jscc",
 16  *   ...
 17  * }</pre>
 18  * 
 19  * <em>NOTE: this is the default grammar engine (if there is no setting in 
 20  * 			 <code>configuration.json</code>, then this engine will be used)
 21  * </em> 
 22  * 
 23  *  <p>
 24  * JS/CC supports grammar generation for:
 25  * LALR(1)
 26  * 
 27  * (in difference to jison, JS/CC supports backtracking to certain extend)
 28  * 
 29  * @see <a href="http://jscc.phorward-software.com/">http://jscc.phorward-software.com/</a>
 30  * 
 31  * @class
 32  * @constant
 33  * @public
 34  * @name JsccGenerator
 35  * @memberOf mmir.env.grammar
 36  * 
 37  * @requires JS/CC
 38  * @requires jQuery.Deferred
 39  * @requires jQuery.extend
 40  * @requires jQuery.ajax
 41  * @requires jQuery.makeArray
 42  */		
 43 function(jscc, constants, configManager, GrammarConverter, $, Logger, module){
 44 
 45 //////////////////////////////////////  template loading / setup for JS/CC generator ////////////////////////////////
 46 
 47 /**
 48  * Deferred object that will be returned - for async-initialization:
 49  * the deferred object will be resolved, when this module has been initialized.
 50  * 
 51  * @private
 52  * @type Deferred
 53  * @memberOf JsccGenerator#
 54  * 
 55  * @see #_loadTemplate
 56  */
 57 var deferred = $.Deferred();
 58 
 59 /**
 60  * The Logger for the JS/CC generator.
 61  * 
 62  * @private
 63  * @type Logger
 64  * @memberOf JsccGenerator#
 65  * 
 66  * @see mmir.Logging
 67  */
 68 var logger = Logger.create(module);
 69 
 70 
 71 //setup logger for compile-messages
 72 /**
 73  * HELPER for creating default logging / error-print functions
 74  * 
 75  * @function
 76  * @private
 77  * @memberOf JsccGenerator#
 78  * 
 79  * @see mmir.Logging
 80  */
 81 function _createCompileLogFunc(log /*Logger*/, level /*String: log-function-name*/, libMakeArray/*helper-lib: provides function makeArray(obj); e.g. jquery*/){
 82 	return function(){
 83 		var args = libMakeArray.makeArray(arguments);
 84 		//prepend "location-information" to logger-call:
 85 		args.unshift('JS/CC', 'compile');
 86 		//output log-message:
 87 		log[level].apply(log, args);
 88 	};
 89 }
 90 
 91 /**
 92  * The URL to the JS/CC template file (generated code-text will be "embedded" in this template)
 93  * 
 94  * @private
 95  * @type String
 96  * @memberOf JsccGenerator#
 97  */
 98 var templatePath = constants.getGrammarPluginPath() + 'grammarTemplate_reduced.tpl';
 99 
100 /**
101  * The argument name when generating the grammar function:
102  * the argument holds the raw text that will be parsed by the generated grammar.
103  * 
104  * NOTE: this argument/variable name must not collide with any code that is generated for the grammar.
105  * 
106  * @constant
107  * @private
108  * @memberOf JsccGenerator#
109  */
110 var INPUT_FIELD_NAME = 'asr_recognized_text';
111 
112 /**
113  * The default options for the JS/CC compiler.
114  * 
115  * To overwrite the default options, configure the following property in <code>www/config/configuration.json</code>:<br>
116  * <pre>
117  * {
118  *   ...
119  *   "grammar": {
120  *   	...
121  *   	"jscc": {
122  *   		"execMode": "your configuration setting!"
123  *   	}
124  *   	...
125  *   },
126  *   ...
127  * }</pre>
128  * 
129  * Valid settings are:
130  * <code>execMode = 'sync' | 'async'</code>
131  * <code>genSourceUrl = true | STRING | FALSY'</code>
132  * 
133  * 
134  * genSourceUrl: if TRUTHY, the sourceUrl for eval'ed parser-module is set
135  *               (i.e. eval'ed code will appear at the URL in debugger, if browser supports sourceURL setting)
136  *               if true: the sourceUrl will be generated using the grammar's ID
137  *               if STRING: the string will be used as sourceUrl; if "<id>" is contained, it will be replaced by the grammar's ID
138  * 
139  * @constant
140  * @private
141  * @default targetType := jscc.MODE_GEN_JS, execMode := sync, genSourceUrl := FALSY
142  * @memberOf JsccGenerator#
143  */
144 var DEFAULT_OPTIONS = {
145 		targetType: jscc.MODE_GEN_JS,
146 		execMode: 'sync',//'sync' | 'async' | default: sync
147 		genSourceUrl: '',// true | STRING: the sourceURL for eval'ed parser-module | default: FALSY 
148 };
149 
150 /**
151  * Name for this plugin/grammar-generator (e.g. used for looking up configuration values in configuration.json).
152  * @constant
153  * @private
154  * @memberOf JsccGenerator#
155  */
156 var pluginName = 'grammar.jison';
157 
158 /**
159  * Exported (public) functions for the JS/CC grammar-engine.
160  * @public
161  * @type GrammarGenerator
162  * @memberOf JsccGenerator#
163  */
164 var jsccGen = {
165 	/** @scope JsccGenerator.prototype */
166 	
167 	/**
168 	 * The name/ID for the compile engine for the JS/CC compiler
169 	 * 
170 	 * @memberOf mmir.env.grammar.JsccGenerator.prototype
171 	 */
172 	engineId: 'jscc',
173 	/**
174 	 * @param {Function} [callback] OPTIONAL
175 	 * 			the callback that is triggered, when the engine is initialized
176 	 * @returns {Deferred}
177 	 * 			a promise that is resolved, when the engine is initialized
178 	 * 			(NOTE: if you use the same function for the <code>callback</code> AND the promise,
179 	 * 			       then the function will be invoked twice!)
180 	 * 
181 	 * @memberOf mmir.env.grammar.JsccGenerator.prototype
182 	 */
183 	init: function(callback){
184 		if(callback){
185 			deferred.always(callback);
186 		}
187 		return deferred;
188 	},
189 	/** @returns {Boolean} if this engine compilation works asynchronously. The current implementation works synchronously (returns FALSE) */
190 	isAsyncCompilation: function(){ return false; },
191 	/**
192 	 * The function for compiling a JSON grammar:
193 	 * 
194 	 * 
195 	 * @param {GrammarConverter} theConverterInstance
196 	 * @param {String} instanceId
197 	 * 				the ID for the compiled grammar (usually this is a language code)
198 	 * @param {Number} fileFormatVersion
199 	 * 				the version of the file format (this is a constant within {@link mmir.SemanticInterpreter}
200 	 * @param callback
201 	 * @returns {GrammarConverter}
202 	 * 			the grammar instance with attached with the compiled function for executing the
203 	 * 			grammar to the instance's {@link GrammarConvert#executeGrammar} property/function. 
204 	 */
205 	compileGrammar: function(theConverterInstance, instanceId, fileFormatVersion, callback){
206         
207 		//attach functions for JS/CC conversion/generation to the converter-instance: 
208 		$.extend(theConverterInstance, JsccGrammarConverterExt);
209 		//attach the JS/CC template to the converter instance
210 		theConverterInstance.TEMPLATE = this.template;
211 		
212 		//start conversion: create grammar in JS/CC syntax (from the JSON definition):
213 		theConverterInstance.init();
214 		this._preparePrintError();
215         theConverterInstance.convertJSONGrammar();
216         var grammarDefinition = theConverterInstance.getGrammarDef();
217 
218         //load options from configuration:
219         var config = configManager.get(pluginName, true, {});
220         //combine with default default options:
221         var options = $.extend({id: instanceId}, DEFAULT_OPTIONS, config);
222         
223         var compileParserModule = function(grammarParserStr, hasError){
224         	
225 	        var addGrammarParserExec = 
226 	    	  '(function(){\n  var semanticInterpreter = require("semanticInterpreter");\n'//FIXME
227 	        	+ 'var options = {fileFormat:'+fileFormatVersion+',execMode:'+JSON.stringify(options.execMode)+'};\n'
228 	        	+ 'var grammarFunc = function('+INPUT_FIELD_NAME+'){'
229 	    			//TODO active/use safe_acc (instead of try-catch construct in semantic-result extraction
230 	//        			+ "var safe_acc = function(obj){\n  \tvar len = arguments.length;\n  \tif(len === 1){\n  \t    return null;\n  \t}\n  \tvar curr = obj, prop = arguments[1], i = 2;\n  \tfor(; i < len; ++i){\n  \tif(obj[prop] != null){\n  \t    obj = obj[prop];\n  \t}\n  \tprop = arguments[i];\n  \t}\n  \tvar res = obj[prop];\n  \treturn typeof res !== 'undefined'? res : null;\n  \t};"
231 	    			+ grammarParserStr
232 	        	+ '\n};\n'
233 	        	
234 	        	//add "self registering" for the grammar-function
235 	        	//  i.e. register the grammar-function for the ID with the SemanticInterpreter
236 	        	+ 'semanticInterpreter.addGrammar("'
237 	        		+instanceId
238 	        		+'", grammarFunc, options);\n\n'
239 	        	+ 'semanticInterpreter.setStopwords("'
240 	        		+instanceId+'",'
241 	        		//store stopwords with their Unicode representation (only for non-ASCII chars)
242 	        		+JSON.stringify(
243 	        				theConverterInstance.getEncodedStopwords()
244 	        		).replace(/\\\\u/gm,'\\u')//<- revert JSON.stringify encoding for the Unicodes
245 	        	+ ');\n'
246 	        	+ 'return grammarFunc;\n'
247 	        	+ '})();'
248 	        ;
249 
250 	        if(options.genSourceUrl){
251             	
252             	var sourceUrlStr;
253             	if(options.genSourceUrl === true){
254             		sourceUrlStr = 'gen/grammar/_compiled_grammar_'+instanceId;
255             	} else {
256             		sourceUrlStr = options.genSourceUrl.toString().replace(/<id>/g,instanceId);
257             	}
258             	
259             	//for Chrome / FireFox debugging: provide an URL for eval'ed code
260             	addGrammarParserExec += '//@ sourceURL='+sourceUrlStr+'\n'
261             							+'//# sourceURL='+sourceUrlStr+'\n';
262                     
263             }
264 	        
265 	        theConverterInstance.setGrammarSource(addGrammarParserExec);
266 	        
267 	        try{
268 	        
269 	        	eval(addGrammarParserExec);
270 	        	
271 	        } catch (err) {
272 	
273 	        	//TODO russa: generate meaningful error message with details about error location
274 	        	//			  eg. use esprima (http://esprima.org) ...?
275 	        	//			... as optional dependency (see deferred initialization above?)
276 	        	
277 	        	var evalMsg = 'Error during eval() for "'+ instanceId +'": ' + err;
278 	        	
279 	        	if(jscc.get_printError()){
280 	        		jscc.get_printError()(evalMsg);
281 	        	}
282 	        	else {
283 	        		logger.error('JS/CC', 'evalCompiled', evalMsg, err);
284 	        	}
285 	        	
286 	        	if(! hasError){
287 	            	evalMsg = '[INVALID GRAMMAR JavaScript CODE] ' + evalMsg;
288 	            	var parseDummyFunc = (function(msg, error){ 
289 	            		return function(){ console.error(msg); console.error(error); throw msg;};
290 	            	})(evalMsg, err);
291 	            	
292 	            	parseDummyFunc.hasErrors = true;
293 	            	
294 	            	theConverterInstance.setGrammarFunction(parseDummyFunc);
295 	        	}
296 	        	
297 	        }
298 	        
299 	        //invoke callback if present:
300 	        if(callback){
301 	        	callback(theConverterInstance);
302 	        }
303 		};
304 		
305 		var isPreventDefault = this._afterCompileParser(compileParserModule, callback);
306         var result = this._compileParser(grammarDefinition, options, isPreventDefault);
307         
308         if(!isPreventDefault){
309         	var hasError = result.hasError;
310         	compileParserModule(result.def, hasError);
311         }
312         
313         return theConverterInstance;
314 	},
315 	/**
316 	 * @protected 
317 	 */
318 	_compileParser: function(grammarDefinition, options, afterCompileParserResult){
319 		
320 		//set up the JS/CC compiler:
321         var dfa_table = '';
322         jscc.reset_all( jscc.EXEC_WEB );
323         
324         var isParsingFailed = false;
325         var grammarParserStr;
326         try {
327         	jscc.parse_grammar(grammarDefinition);
328         } catch (err){
329         	var msg = 'Error while compiling grammar: ' + (err.stack?err.stack:err);
330         	isParsingFailed = msg;
331         	
332         	msg = '[INVALID GRAMMAR] ' + msg;
333         	grammarParserStr = 'var msg = '+JSON.stringify(msg)+'; console.error(msg); throw msg;';//FIXME
334         }
335       
336         if (jscc.getErrors() == 0) {
337         	jscc.undef();
338         	jscc.unreachable();
339                 
340             if (jscc.getErrors() == 0) {
341             	jscc.first();
342             	jscc.print_symbols();
343             	dfa_table = jscc.create_subset(jscc.get_nfa_states());
344             	dfa_table = jscc.minimize_dfa(dfa_table);
345             	
346             	jscc.set_dfa_table(dfa_table);//FIXME: check, if this is really necessary
347                 
348             	jscc.lalr1_parse_table(false);
349             	jscc.resetErrors();
350             }
351         }
352      
353         if (jscc.getErrors() > 0 || jscc.getWarnings() > 0){
354             logger.error(
355             		'JSCC', 'compile', 'there occured' 
356             		+ (jscc.getWarnings() > 0? jscc.getWarnings() + ' warning(s)' : '')
357             		+ (jscc.getErrors() > 0? jscc.getErrors() + ' error(s)' : '')
358             		+ ' during compilation.'
359             );
360         }
361         jscc.resetErrors();
362         jscc.resetWarnings();
363         
364 //      console.debug("before replace " + theConverterInstance.PARSER_TEMPLATE);//debug
365      
366         if( ! isParsingFailed){
367         	
368         	var genData = this._getGenerated(options.targetType, dfa_table);
369         	grammarParserStr = this._applyGenerated(genData, this.template);
370         }
371 		
372 		return {def: grammarParserStr, hasError: isParsingFailed};
373 	},
374 	/**
375 	 * @protected
376 	 */
377 	_preparePrintError: function(){
378 
379 		//only set print-message function, if it is not already set 
380 		//  (i.e. if JS/CC still has its default print function set)
381 		if(jscc.is_printError_default()){
382 			/**
383 			 * The default logging function for printing compilation errors.
384 			 * @private
385 			 * @name set_printError
386 			 * @function
387 			 * @memberOf JsccGenerator.jscc#
388 			 */
389 			jscc.set_printError(	_createCompileLogFunc(logger, 'error', $));
390 		}
391 		if(jscc.is_printWarning_default()){
392 			/**
393 			 * The default logging function for printing compilation warnings.
394 			 * @private
395 			 * @name set_printWarning
396 			 * @function
397 			 * @memberOf JsccGenerator.jscc#
398 			 */
399 			jscc.set_printWarning(	_createCompileLogFunc(logger, 'warn', $));
400 		}
401 		if(jscc.is_printInfo_default()){
402 			/**
403 			 * The default logging function for printing compilation information.
404 			 * @private
405 			 * @name set_printInfo
406 			 * @function
407 			 * @memberOf JsccGenerator.jscc#
408 			 */
409 			jscc.set_printInfo(		_createCompileLogFunc(logger, 'info', $));
410 		}
411 	},
412 	/**
413 	 * The default logging / error-print function for JS/CC.
414 	 * 
415 	 * @protected
416 	 * 
417 	 * @see mmir.Logging
418 	 */
419 	printError: function(){
420 		var errorFunc = jscc.get_printError();
421 		if(errorFunc){
422 			errorFunc.apply(jscc, arguments);
423 		} else {
424 			var args = $.makeArray(arguments);
425 			//prepend "location-information" to logger-call:
426 			args.unshift('JS/CC', 'compile');
427 			//output log-message:
428 			logger.error.apply(logger, args);
429 		}
430 	},
431 	/**
432 	 * Optional hook for pre-processing the generated parser, after the parser is generated.
433 	 * 
434 	 * By default, this function returns VOID, in which case the parser-module is created by default.
435 	 * 
436 	 * If a function is returned instead, then it must invoke <code>compileParserModuleFunc</code>:
437 	 * <code>compileParserModuleFunc(compiledParser : STRING, hasErrors : BOOLEAN)</code>
438 	 * 
439 	 * 
440 	 * @param {Function} compileParserModuleFunc
441 	 * 				the function that generates the parser-module:
442 	 * 				<code>compileParserModuleFunc(compiledParser : STRING, hasErrors : BOOLEAN)</code>
443 	 * 
444 	 * @param {Function} compileCallbackFunc
445 	 * 				the callback function which will be invoked by compileParserModuleFunc, after it has finished.
446 	 * 				If compileParserModuleFunc() is prevented from exectution then the callback MUST be invoked manually
447 	 * 				<code>compileCallbackFunc(theConverterInstance: GrammarConverter)</code>
448 	 * 
449 	 * @returns {TRUTHY|VOID}
450 	 * 				FALSY for the default behavior.
451 	 * 				IF a TRUTHY value is returned, then the default action after compiling the parser
452 	 * 				is not executed:
453 	 * 					i.e. compileParserModuleFunc is not automatically called and in consequence the callback is not invoked
454 	 * 					
455 	 * 				
456 	 * 				NOTE: if not FALSY, then either compileParserModuleFunc() must be invoked, or the callback() must be invoked!
457 	 * 
458 	 * @protected
459 	 */
460 	_afterCompileParser: function(compileParserModuleFunc, compileCallbackFunc){
461 		//default: return VOID
462 		return;
463 	},
464 	/**
465 	 * @protected
466 	 */
467 	_getGenerated: function(genMode, dfaTable){
468 		return {
469 			head: jscc.get_code_head(),
470 			tables: jscc.print_parse_tables(genMode),//jscc.MODE_GEN_JS
471 			dfa: jscc.print_dfa_table(dfaTable),//dfa_table
472 			terminals: jscc.print_term_actions(),
473 			labels: jscc.print_symbol_labels(),
474 			actions: jscc.print_actions(),
475 			error: jscc.get_error_symbol_id(),
476 			eof: jscc.get_eof_symbol_id(),
477 			whitespace: jscc.get_whitespace_symbol_id()
478 		};
479 	},
480 	/**
481 	 * @protected
482 	 */
483 	_applyGenerated: function(genData, template){
484     	
485         var grammarParserStr = template;
486         grammarParserStr = grammarParserStr.replace(/##PREFIX##/g, "");
487         grammarParserStr = grammarParserStr.replace(/##HEADER##/g, genData.head);
488         grammarParserStr = grammarParserStr.replace(/##TABLES##/g, genData.tables);
489         grammarParserStr = grammarParserStr.replace(/##DFA##/g, genData.dfa);
490         grammarParserStr = grammarParserStr.replace(/##TERMINAL_ACTIONS##/g, genData.terminals);
491         grammarParserStr = grammarParserStr.replace(/##LABELS##/g, genData.labels);
492         grammarParserStr = grammarParserStr.replace(/##ACTIONS##/g, genData.actions);
493         grammarParserStr = grammarParserStr.replace(/##FOOTER##/g, "\nvar _semanticAnnotationResult = { result: {}};\n__parse( "+INPUT_FIELD_NAME+", new Array(), new Array(), _semanticAnnotationResult);\nreturn _semanticAnnotationResult.result;");
494         grammarParserStr = grammarParserStr.replace(/##ERROR##/g, genData.error);
495         grammarParserStr = grammarParserStr.replace(/##EOF##/g, genData.eof);
496         grammarParserStr = grammarParserStr.replace(/##WHITESPACE##/g, genData.whitespace);
497         
498         return grammarParserStr;
499 	}
500 };
501 
502 /**
503  * Initializes the grammar-engine:
504  * 
505  * loads the template and resolves the engine as initialzed.
506  * 
507  * @param {String} url
508  * 			URL to the template file
509  * @param {GrammarGenerator} jsccGenerator
510  * 			the JS/CC grammar generator instance
511  * @param {Deferred} promise
512  * 			the {@link #deferred} for resolving the generator
513  * 			as initialized.
514  *  
515  * 
516  * @private
517  * @function
518  * @memberOf JsccGenerator#
519  */
520 function _loadTemplate(url, jsccGenerator, promise){
521 	$.ajax({
522 		url: url,
523 		dataType: 'text',
524 		async: true,
525 		success: function(data){
526 			
527 			jsccGenerator.template = data;
528 			
529 			promise.resolve();
530 			
531 		},
532 		error: function(xhr, status, err){
533 			var msg = 'Failed to load grammar template file from "'+templatePath+'": '+status+', ERROR '+err;
534 			console.error(msg);
535 			promise.reject(msg);
536 		}
537 	});
538 }
539 
540 //load the JS/CC template and resolve this module as "initialzed":
541 _loadTemplate(templatePath, jsccGen, deferred);
542 
543 ////////////////////////////////////// JS/CC specific extensions to GrammarConverter ////////////////////////////////
544 
545 /**
546  * JS/CC specific extension / implementation for {@link GrammarConverter} instances
547  * 
548  * @type GrammarConverter
549  * @memberOf JsccGenerator#
550  */
551 var JsccGrammarConverterExt = {
552 	/** @memberOf JsccGrammarConverterExt */
553 	init: function(){
554 
555 		this.THE_INTERNAL_GRAMMAR_CONVERTER_INSTANCE_NAME = "theGrammarConverterInstance";
556 		this.grammar_tokens = "/~ --- Token definitions --- ~/\n\n/~ Characters to be ignored ~/\n!   ' |\\t' ;\n\n/~ Non-associative tokens ~/\n";
557 		this.grammar_utterances = "";
558 		this.grammar_phrases = "phrases:";
559 		this.token_variables = "[*\n  var " + this.variable_prefix
560 				+ "result = '';\n";
561 		this.tokens_array = [];
562 	},
563 	convertJSONGrammar: function(){
564 	
565 		this.json_grammar_definition = this.maskJSON(this.json_grammar_definition);
566 		
567 		this.parseTokens();
568 		this.parseUtterances();
569 		this.parseStopWords();
570 		
571 		this.jscc_grammar_definition = this.token_variables
572 				+ "*]\n\n"
573 				+ this.grammar_tokens
574 				+ "\n\n ##\n\n/~ --- Grammar specification --- ~/\n\nutterance:      phrases    [*  "
575 				
576 				//TODO use LOG LEVEL for activating / deactivating this:
577 				+ "console.log("
578 				+ this.variable_prefix + "result); "
579 				
580 				+ "semanticAnnotationResult.result = "
581 				+ this.variable_prefix + "result; *] ;\n\n" + this.grammar_utterances
582 				+ "\n" + this.grammar_phrases + ";";
583 	
584 		this.json_grammar_definition = this.unmaskJSON(this.json_grammar_definition);
585 	},
586 	parseTokens: function(){
587 		var self = this;
588 		var json_tokens =  this.json_grammar_definition.tokens;
589 		var pref = self.variable_prefix;
590 		
591 		for(token_name in json_tokens){
592 			
593 			var words = json_tokens[token_name];
594 			
595 			self.token_variables += "  var " + pref
596 					+ token_name.toLowerCase() + " = {};\n";
597 			
598 			var grammar_token ="    '";
599 			
600 			for(var i=0, size = words.length; i < size ; ++i){
601 				if(i > 0){
602 					grammar_token += "|";
603 				}
604 				grammar_token += words[i];
605 			}
606 			
607 			grammar_token += "'    " + token_name + " [* " + pref
608 					+ token_name.toLowerCase() + "[%match] = %match; *];\n";
609 			
610 			self.grammar_tokens += grammar_token;
611 		}
612 	},
613 	parseUtterances: function(){
614 		var self = this;
615 		var utt_index = 0;
616 		var json_utterances =  this.json_grammar_definition.utterances;
617 	
618 		for(var utterance_name in json_utterances){
619 			var utterance_def = json_utterances[utterance_name];
620 			if(utt_index > 0){
621 				self.grammar_phrases += "\n\t|";
622 			}
623 			utt_index++;
624 			self.doParseUtterance(utterance_name, utterance_def);
625 		}
626 	},
627 	doParseUtterance: function(utterance_name, utterance_def){
628 		var grammar_utterance = utterance_name + ":";
629 		var self = this; 
630 		self.token_variables += "  var " + self.variable_prefix
631 				+ utterance_name.toLowerCase() + " = {};\n";
632 		//self.grammar_phrases += utterance_name + "  " +  self.doCreateSemanticInterpretationForUtterance(utterance_name, utterance_def);
633 		self.grammar_phrases += utterance_name + "  " ;
634 		var phrases = utterance_def.phrases;
635 		var semantic  = self.doCreateSemanticInterpretationForUtterance(utterance_name, utterance_def);
636 		
637 		for(var index=0,size=phrases.length; index < size; ++index){
638 			if(index > 0){
639 				grammar_utterance += "\n|";
640 			}
641 			var phrase = phrases[index];
642 			var semantic_interpretation = self.doCreateSemanticInterpretationForPhrase(
643 					utterance_name.toLowerCase(), utterance_def, phrase, semantic
644 			);
645 			grammar_utterance += phrase + semantic_interpretation;
646 		}
647 		self.grammar_utterances += grammar_utterance + ";\n\n";
648 	},
649 	doCreateSemanticInterpretationForUtterance: function(utterance_name, utterance_def){
650 		var semantic = utterance_def.semantic,
651 		variable_index, variable_name;
652 		
653 		if(logger.isDebug()) logger.debug('doCreateSemanticInterpretationForUtterance: '+semantic);//debug
654 		
655 		var semantic_as_string = JSON.stringify(semantic);
656 		if( semantic_as_string != null){
657 		this.variable_regexp.lastIndex = 0;
658 		var variables = this.variable_regexp.exec(semantic_as_string);
659 		while (variables != null) {
660 			var variable = variables[1],
661 			remapped_variable_name = "";
662 			
663 			if(logger.isDebug()) logger.debug("variables " + variable, semantic_as_string);//debug
664 			
665 			variable_index = /\[(\d+)\]/.exec(variable);
666 			variable_name = new RegExp('_\\$([a-zA-Z_][a-zA-Z0-9_\\-]*)').exec(variable)[1];
667 //			variableObj = /_\$([a-zA-Z_][a-zA-Z0-9_\-]*)(\[(\d+)\])?(\["semantic"\]|\['semantic'\]|\.semantic)?/.exec(variable);
668 //			variableObj = /_\$([a-zA-Z_][a-zA-Z0-9_\-]*)(\[(\d+)\])?((\[(("(.*?[^\\])")|('(.*?[^\\])'))\])|(\.(\w+)))?/.exec(variable);
669 	//"_$NAME[INDEX]['FIELD']":  _$NAME                  [ INDEX ]        [" FIELD "]  | [' FIELD ']      |   .FIELD
670 			if (variable_index == null) {
671 				remapped_variable_name = variable;
672 			} else {
673 					remapped_variable_name = variable.replace(
674 							  '[' + variable_index[1] + ']'
675 							, "["
676 								+ utterance_name.toLowerCase() + "_temp['phrases']['"
677 								+ variable_name.toLowerCase() + "']["
678 								+ variable_index[1]
679 							+ "]."+this.entry_token_field+"]");
680 					//TODO replace try/catch with safe_acc function
681 					//     PROBLEM: currently, the format for variable-access is not well defined
682 					//              -> in case of accessing the "semantic" field for a variable reference of another Utterance
683 					//                 we would need another safe_acc call 
684 					//				   ... i.e. need to parse expression for this, but since the format is not well defined
685 					//				   we cannot say, for what exactly we should parse...
686 					//                 NORMAL VAR EXPR: 		_$a_normal_token[0]
687 					//                 ACCESS TO SEMANTICS: 	_$other_utterance[0]['semantic']
688 					//                                      but this could also be expressed e.g. as _$other_utterance[0].semantic
689 					//                                      ...
690 //					remapped_variable_name = variable.replace(
691 //							  '[' + variable_index[1] + ']'
692 //							, "[safe_acc("
693 //								+ utterance_name.toLowerCase() + "_temp, 'phrases', '"
694 //								+ variable_name.toLowerCase() + "', "
695 //								+ variable_index[1] 
696 //								+ ")]"
697 //							);
698 			}
699 			semantic_as_string = semantic_as_string.replace(
700 					variables[0],
701 					" function(){try{return " + remapped_variable_name
702 						+ ";} catch(e){return void(0);}}() "
703 //					"' + " + remapped_variable_name + " + '"//TODO replace try/catch with safe_acc function
704 			);
705 			variables =  this.variable_regexp.exec(semantic_as_string);
706 		}
707 		}
708 		return semantic_as_string;
709 	},
710 	doCreateSemanticInterpretationForPhrase: function(utterance_name, utterance_def, phrase, semantic_as_string){
711 		var phraseList = phrase.split(/\s+/),
712 		length = phraseList.length,
713 		duplicate_helper = {};
714 		
715 		var result = " [* %% = ";
716 		var i = 0;
717 		while (i < length){
718 			i++;
719 			result += "%"+i;
720 			if(i < length){
721 				result += " + ' ' + ";
722 			}
723 		}
724 		result += "; var "+utterance_name+"_temp = {}; "+utterance_name+"_temp['phrases'] = {};";
725 		for (i = 0; i < length; i += 1) {
726 			if (typeof(duplicate_helper[phraseList[i]]) == "undefined") {
727 				duplicate_helper[phraseList[i]] = 0;
728 				result += utterance_name+"_temp['phrases']['"+phraseList[i].toLowerCase()+"'] = [];";
729 			} else {
730 				duplicate_helper[phraseList[i]] += 1;
731 			}
732 			result += utterance_name + "_temp['phrases']['"
733 						+ phraseList[i].toLowerCase() + "']["
734 						+ duplicate_helper[phraseList[i]] + "] = {"
735 							+ this.entry_token_field + ": %" + (i + 1) + ","
736 							+ this.entry_index_field + ": " + i
737 						+"};\n\t\t";
738 		}
739 		result += "var " + this.variable_prefix + "phrase = %%; " 
740 				+ utterance_name + "_temp['phrase']=" + this.variable_prefix + "phrase; "
741 				+ utterance_name + "_temp['utterance']='" + utterance_name + "'; "
742 				+ utterance_name + "_temp['engine']='jscc'; "//FIXME debug
743 				+ utterance_name + "_temp['semantic'] = " + semantic_as_string
744 				+ "; " + this.variable_prefix + utterance_name + "["
745 				+ this.variable_prefix + "phrase] = " + utterance_name + "_temp; "
746 				+ this.variable_prefix + "result = " + utterance_name + "_temp; *]";
747 		return result;
748 	}
749 };
750 
751 
752 return jsccGen;
753 
754 });