4 attr_accessor :arguments, :token_index
6 def initialize(arguments)
8 @arguments = arguments.split("")
9 @seperation_counter = 0
13 # Scans the input string and identifies single language tokens
15 if @token_index >= @arguments.size
19 case @arguments[@token_index]
27 if (@arguments[@token_index + 1] == "o") && (@arguments[@token_index + 2] == "t") && ((@arguments[@token_index + 3] == " ") || (@arguments[@token_index + 3] == "("))
38 if (@arguments[@token_index + 1] == "n") && (@arguments[@token_index + 2] == "d") && ((@arguments[@token_index + 3] == " ") || (@arguments[@token_index + 3] == "("))
46 if (@arguments[@token_index + 1] == "r") && ((@arguments[@token_index + 2] == " ") || (@arguments[@token_index + 2] == "("))
62 # Helper generates a statement token
65 current_token_value = ""
69 if (@arguments[j] == "/")
71 current_token_value << @arguments[j]
73 end until (j >= @arguments.size) || (@arguments[j] =~ /\s/)
74 elsif (@arguments[j] =~ /=|<|>/)
75 while !(@arguments[j] =~ /=|<|>/)
76 current_token_value << @arguments[j]
80 current_token_value << @arguments[j]
83 if @arguments[j] == "/"
85 current_token_value << @arguments[j]
87 if @arguments[j] == "/"
88 current_token_value << "/"
91 end until (j >= @arguments.size) || (@arguments[j] =~ /\//)
93 while (j < @arguments.size) && ((@arguments[j] != " ") && (@arguments[j] != ")"))
94 current_token_value << @arguments[j]
100 # Identify and tokenize regular expressions by ignoring everything between /'s
101 if @arguments[j] == '/'
102 current_token_value << '/'
104 while(j < @arguments.size && @arguments[j] != '/')
105 current_token_value << @arguments[j]
108 current_token_value << @arguments[j] if @arguments[j]
111 if @arguments[j+1] == "("
115 current_token_value << @arguments[j]
117 while !(j+1 >= @arguments.size) && @arguments[j] != ')'
119 current_token_value << @arguments[j]
126 if(@arguments[j] == ' ')
127 break if(is_klass?(j) && !(@arguments[j-1] =~ /=|<|>/))
129 if( (@arguments[j] == ' ') && (@seperation_counter < 2) && !(current_token_value.match(/^.+(=|<|>).+$/)) )
130 if((index = lookahead(j)))
134 end until (j >= @arguments.size) || (@arguments[j] =~ /\s|\)/)
135 @seperation_counter = 0
137 rescue Exception => e
138 raise "An exception was raised while trying to tokenize '#{current_token_value} - #{e}'"
141 @token_index += current_token_value.size + @white_spaces - 1
145 if current_token_value.match(/.+?\($/)
146 return "bad_token", [@token_index - current_token_value.size + 1, @token_index]
148 elsif current_token_value.match(/^\/.+?\/(<|>|=).+/)
149 return "bad_token", [@token_index - current_token_value.size + 1, @token_index]
150 elsif current_token_value.match(/^.+?\/(<|>|=).+/)
151 return "bad_token", [@token_index - current_token_value.size + 1, @token_index]
154 if current_token_value.match(/^.+?\((\s*(')[^']*(')\s*(,\s*(')[^']*('))*)?\)(\.[a-zA-Z0-9_]+)?((!=|<=|>=|=|>|<).+)?$/) ||
155 current_token_value.match(/^.+?\((\s*(")[^"]*(")\s*(,\s*(")[^"]*("))*)?\)(\.[a-zA-Z0-9_]+)?((!=|<=|>=|=|>|<).+)?$/)
156 return "fstatement", current_token_value
158 return "bad_token", [@token_index - current_token_value.size + 1, @token_index]
162 current_token_value.split('').each do |c|
164 slash_err = !slash_err
167 return "bad_token", [@token_index - current_token_value.size + 1, @token_index] if slash_err
168 return "statement", current_token_value
173 # Deal with special puppet class statement
175 while(j < @arguments.size && @arguments[j] == ' ')
179 if @arguments[j] =~ /=|<|>/
186 # Eat spaces while looking for the next comparison symbol
189 while(index <= @arguments.size)
191 unless(@arguments[index] =~ /\s/)
192 @seperation_counter +=1