| CODENOTIFIER | HelpYou are not signed inSign in |
Project: Jython
Revision: 5288
Author: fwierzbicki
Date: 04 Sep 2008 12:57:30
Changes:Fix for some offset problems -- the biggest one being dedents now take on the
offset values of the previous token.
| ...... | @@ -157,14 +157,18 @@ | |
| 157 | 157 | stream.consume(); |
| 158 | 158 | |
| 159 | 159 | if (t.getType() == Token.EOF) { |
| 160 | Token prev = stream.LT(-1); | |
| 160 | 161 | if (!inSingle) { |
| 161 | Token prev = stream.LT(-1); | |
| 162 | 162 | if (prev == null || prev.getType() != PythonLexer.NEWLINE) { |
| 163 | 163 | generateNewline(t); |
| 164 | 164 | } |
| 165 | 165 | } |
| 166 | ||
| 167 | handleDedents(-1, (CommonToken)t); | |
| 166 | if (prev != null) { | |
| 167 | ((CommonToken)t).setStopIndex(((CommonToken)prev).getStopIndex()); | |
| 168 | handleDedents(-1, (CommonToken)prev); | |
| 169 | } else { | |
| 170 | handleDedents(-1, (CommonToken)t); | |
| 171 | } | |
| 168 | 172 | enqueue(t); |
| 169 | 173 | } else if (t.getType() == PythonLexer.NEWLINE) { |
| 170 | 174 | // save NEWLINE in the queue |
| ...... | @@ -203,7 +207,12 @@ | |
| 203 | 207 | handleIndents(cpos, (CommonToken)t); |
| 204 | 208 | } |
| 205 | 209 | else if (cpos < lastIndent) { // they dedented |
| 206 | handleDedents(cpos, (CommonToken)t); | |
| 210 | Token prev = stream.LT(-1); | |
| 211 | if (prev != null) { | |
| 212 | handleDedents(cpos, (CommonToken)prev); | |
| 213 | } else { | |
| 214 | handleDedents(cpos, (CommonToken)t); | |
| 215 | } | |
| 207 | 216 | } |
| 208 | 217 | |
| 209 | 218 | if (t.getType() == Token.EOF && inSingle) { |
| ...... | @@ -12,6 +12,8 @@ | |
| 12 | 12 | if (t==null) { |
| 13 | 13 | return; |
| 14 | 14 | } |
| 15 | //System.out.println("setting boundries on '"+t+"' with start: '" + startToken + "' and '" + | |
| 16 | // stopToken + "'"); | |
| 15 | 17 | int start = 0; |
| 16 | 18 | int stop = 0; |
| 17 | 19 | int startChar = 0; |
| ...... | @@ -284,6 +284,12 @@ | |
| 284 | 284 | List stypes = new ArrayList(); |
| 285 | 285 | } |
| 286 | 286 | @after { |
| 287 | if (!stypes.isEmpty()) { | |
| 288 | //The EOF token messes up the end offsets, so set them manually. | |
| 289 | PythonTree stop = (PythonTree)stypes.get(stypes.size() -1); | |
| 290 | mtype.setCharStopIndex(stop.getCharStopIndex()); | |
| 291 | mtype.setTokenStopIndex(stop.getTokenStopIndex()); | |
| 292 | } | |
| 287 | 293 | $file_input.tree = mtype; |
| 288 | 294 | } |
| 289 | 295 | : (NEWLINE |