Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit f9fd287

Browse files
committed
refactoring of comments processing during read
1 parent 21c8b2a commit f9fd287

File tree

4 files changed

+139
-52
lines changed

4 files changed

+139
-52
lines changed

‎src/main/java/com/igormaznitsa/prologparser/PrologParser.java

Lines changed: 95 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -187,58 +187,84 @@ public ParserContext getContext() {
187187
return context;
188188
}
189189

190-
public boolean hasNext() {
191-
return this.tokenizer.peek() != null;
190+
private PrologTermReadResult deferredReadTerm;
191+
192+
private static boolean isComment(final TokenizerResult tokenizerResult) {
193+
return tokenizerResult != null
194+
&& tokenizerResult.getResult().getType() == TermType.ATOM
195+
&& (tokenizerResult.getResult().getQuotation() == Quotation.COMMENT_LINE
196+
|| tokenizerResult.getResult().getQuotation() == Quotation.COMMENT_BLOCK);
197+
}
198+
199+
private void notifyCommentTokenListeners(final TokenizerResult commentToken) {
200+
for (final TokenizedCommentListener listener : this.commentTokenListeners) {
201+
listener.onCommentToken(this, commentToken);
202+
}
192203
}
193204

194-
private TokenizerResult readNextTokenCommentAware() {
205+
private TokenizerResult extractNextTokenCommentAware() {
195206
TokenizerResult result;
196-
if (this.commentsAsAtoms) {
197-
while (true) {
198-
result = this.tokenizer.readNextToken();
199-
if (result != null
200-
&& (result.getResult().getQuotation() == Quotation.COMMENT_BLOCK ||
201-
result.getResult().getQuotation() == Quotation.COMMENT_LINE)) {
202-
for (final TokenizedCommentListener listener : this.commentTokenListeners) {
203-
listener.onCommentToken(this, result);
204-
}
205-
} else {
206-
break;
207-
}
208-
}
209-
} else {
207+
while (true) {
210208
result = this.tokenizer.readNextToken();
209+
if (isComment(result)) {
210+
this.notifyCommentTokenListeners(result);
211+
} else {
212+
break;
213+
}
211214
}
212215
return result;
213216
}
214217

215-
public PrologTerm next() {
216-
final PrologTerm found = readBlock(OPERATORS_PHRASE);
217-
if (found == null) {
218-
throw new NoSuchElementException("No terms in source");
219-
} else {
220-
final TokenizerResult endAtom = this.readNextTokenCommentAware();
221-
if (endAtom == null || !endAtom.getResult().getText().equals(OPERATOR_DOT.getText())) {
222-
throw new PrologParserException("End operator is not found",
223-
this.tokenizer.getLine(),
224-
this.tokenizer.getPos());
218+
private PrologTermReadResult extractNextBlockAndWrapError() {
219+
try {
220+
final PrologTerm found = this.readBlock(OPERATORS_PHRASE);
221+
if (found == null) {
222+
throw new NoSuchElementException("No terms in source");
223+
} else {
224+
final TokenizerResult endAtom = this.extractNextTokenCommentAware();
225+
if (endAtom == null || !endAtom.getResult().getText().equals(OPERATOR_DOT.getText())) {
226+
throw new PrologParserException("End operator is not found",
227+
this.tokenizer.getLine(),
228+
this.tokenizer.getPos());
229+
}
225230
}
231+
return new PrologTermReadResult(found, null);
232+
} catch (RuntimeException ex) {
233+
return new PrologTermReadResult(null, ex);
234+
}
235+
}
236+
237+
public boolean hasNext() {
238+
if (this.deferredReadTerm == null) {
239+
this.deferredReadTerm = this.extractNextBlockAndWrapError();
226240
}
227-
return found;
241+
return this.deferredReadTerm.isPresented();
242+
}
243+
244+
public PrologTerm next() {
245+
if (this.deferredReadTerm == null) {
246+
this.deferredReadTerm = this.extractNextBlockAndWrapError();
247+
}
248+
final PrologTerm result;
249+
try {
250+
result = this.deferredReadTerm.getResult();
251+
} finally {
252+
this.deferredReadTerm = null;
253+
}
254+
return result;
228255
}
229256

230257
private PrologStruct readStruct(final PrologTerm functor) {
231258
final List<PrologTerm> listOfAtoms = new ArrayList<>();
232259
PrologStruct result;
233260
boolean active = true;
234261
while (active) {
235-
final PrologTerm block = readBlock(OPERATORS_INSIDE_STRUCT);
236-
262+
final PrologTerm block = this.readBlock(OPERATORS_INSIDE_STRUCT);
237263
if (block == null) {
238264
return null;
239265
}
240266

241-
final TokenizerResult nextAtom = this.readNextTokenCommentAware();
267+
final TokenizerResult nextAtom = this.extractNextTokenCommentAware();
242268
if (nextAtom == null) {
243269
throw new PrologParserException("Can't read next token in block", this.tokenizer.getLine(),
244270
this.tokenizer.getPos());
@@ -277,7 +303,7 @@ private PrologTerm readList(final TokenizerResult openingBracket) {
277303
while (continueReading) {
278304
final PrologTerm block = readBlock(OPERATORS_INSIDE_LIST);
279305

280-
final TokenizerResult nextAtom = this.readNextTokenCommentAware();
306+
final TokenizerResult nextAtom = this.extractNextTokenCommentAware();
281307
if (nextAtom == null) {
282308
throw new PrologParserException("Can't read next token in list", this.tokenizer.getLine(),
283309
this.tokenizer.getPos());
@@ -316,7 +342,7 @@ private PrologTerm readList(final TokenizerResult openingBracket) {
316342
tokenizer.getLastTokenPos(), null);
317343
}
318344

319-
final TokenizerResult nextAtomTwo = this.readNextTokenCommentAware();
345+
final TokenizerResult nextAtomTwo = this.extractNextTokenCommentAware();
320346
if (nextAtomTwo == null) {
321347
throw new PrologParserException("Can't find expected token in list",
322348
this.tokenizer.getLine(), this.tokenizer.getPos());
@@ -374,21 +400,14 @@ private PrologTerm readList(final TokenizerResult openingBracket) {
374400
return leftPartFirst;
375401
}
376402

377-
private void checkForNull(final Object obj, final String message,
378-
final TokenizerResult startTerm) {
379-
if (obj == null) {
380-
throw new PrologParserException(message, startTerm.getLine(), startTerm.getPos());
381-
}
382-
}
383-
384403
private PrologTerm readBlock(final Koi7CharOpMap endOperators) {
385404
// the variable will contain last processed tree item contains either
386405
// atom or operator
387406
AstItem currentTreeItem = null;
388407

389408
while (true) {
390409
// read next atom from tokenizer
391-
TokenizerResult readAtomContainer = this.readNextTokenCommentAware();
410+
TokenizerResult readAtomContainer = this.extractNextTokenCommentAware();
392411

393412
if (readAtomContainer == null) {
394413
if (currentTreeItem == null) {
@@ -505,7 +524,7 @@ private PrologTerm readBlock(final Koi7CharOpMap endOperators) {
505524
readAtomContainer.getLine(), readAtomContainer.getPos());
506525
}
507526

508-
final TokenizerResult token = this.readNextTokenCommentAware();
527+
final TokenizerResult token = this.extractNextTokenCommentAware();
509528

510529
final PrologTerm closingAtom;
511530
if (token == null) {
@@ -534,7 +553,7 @@ private PrologTerm readBlock(final Koi7CharOpMap endOperators) {
534553
}
535554
} else {
536555
if (readAtom.getType() != TermType.VAR || (this.parserFlags & FLAG_VAR_AS_FUNCTOR) != 0) {
537-
TokenizerResult nextToken = this.readNextTokenCommentAware();
556+
TokenizerResult nextToken = this.extractNextTokenCommentAware();
538557

539558
if (nextToken == null) {
540559
throw new PrologParserException("Non-closed clause", this.tokenizer.getLastTokenLine(),
@@ -666,6 +685,19 @@ private PrologTerm readBlock(final Koi7CharOpMap endOperators) {
666685
}
667686
}
668687

688+
private void checkForNull(final Object obj, final String message,
689+
final TokenizerResult startTerm) {
690+
if (obj == null) {
691+
throw new PrologParserException(message, startTerm.getLine(), startTerm.getPos());
692+
}
693+
}
694+
695+
@Override
696+
public void close() throws IOException {
697+
this.deferredReadTerm = null;
698+
this.tokenizer.close(this.autoCloseReaderFlag);
699+
}
700+
669701
@Override
670702
public Iterator<PrologTerm> iterator() {
671703
return new Iterator<>() {
@@ -690,9 +722,26 @@ public Stream<PrologTerm> stream() {
690722
);
691723
}
692724

693-
@Override
694-
public void close() throws IOException {
695-
this.tokenizer.close(this.autoCloseReaderFlag);
725+
private static final class PrologTermReadResult {
726+
private final PrologTerm result;
727+
private final RuntimeException exception;
728+
729+
private PrologTermReadResult(final PrologTerm result, final RuntimeException error) {
730+
this.result = result;
731+
this.exception = error;
732+
}
733+
734+
boolean isPresented() {
735+
return this.exception == null || !(this.exception instanceof NoSuchElementException);
736+
}
737+
738+
PrologTerm getResult() {
739+
if (this.exception == null) {
740+
return this.result;
741+
} else {
742+
throw this.exception;
743+
}
744+
}
696745
}
697746

698747
}

‎src/main/java/com/igormaznitsa/prologparser/tokenizer/Tokenizer.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -250,8 +250,8 @@ public void push(final TokenizerResult object) {
250250
public TokenizerResult peek() {
251251
TokenizerResult result;
252252
if (this.lastPushedTerm == null) {
253-
result = readNextToken();
254-
push(result);
253+
result = this.readNextToken();
254+
this.push(result);
255255
} else {
256256
result = this.lastPushedTerm;
257257
}
@@ -332,7 +332,7 @@ public TokenizerResult pop() {
332332
public TokenizerResult readNextToken() {
333333

334334
if (this.lastPushedTerm != null) {
335-
return pop();
335+
return this.pop();
336336
}
337337

338338
Quotation quoting = Quotation.NONE;

‎src/test/java/com/igormaznitsa/prologparser/IntegrationTest.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,13 +73,13 @@ void testReadingClausesTokensAndCharsFromSameParser() throws Exception {
7373
assertEquals("lsome(a)", term1.toString());
7474
assertTrue(parser.hasNext());
7575
final TokenizerResult token1 = parser.getInternalTokenizer().readNextToken();
76-
assertEquals("alone", Objects.requireNonNull(token1).getResult().toString());
76+
assertEquals("next", Objects.requireNonNull(token1).getResult().toString());
7777
assertTrue(parser.hasNext());
7878
final TokenizerResult token2 = parser.getInternalTokenizer().readNextToken();
79-
assertEquals(".", Objects.requireNonNull(token2).getResult().getText());
79+
assertEquals("(", Objects.requireNonNull(token2).getResult().getText());
8080
assertTrue(parser.hasNext());
8181
final PrologTerm term2 = parser.next();
82-
assertEquals("next(b)", term2.toString());
82+
assertEquals("alone", term2.toString());
8383
assertTrue(parser.hasNext());
8484
assertThrows(PrologParserException.class, parser::next);
8585
}

‎src/test/java/com/igormaznitsa/prologparser/exceptions/PrologParserExceptionTest.java

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,16 @@
44
import static org.junit.jupiter.api.Assertions.assertFalse;
55
import static org.junit.jupiter.api.Assertions.assertTrue;
66

7+
import com.igormaznitsa.prologparser.DefaultParserContext;
8+
import com.igormaznitsa.prologparser.GenericPrologParser;
9+
import com.igormaznitsa.prologparser.ParserContext;
10+
import com.igormaznitsa.prologparser.PrologParser;
11+
import com.igormaznitsa.prologparser.terms.PrologTerm;
12+
import com.igormaznitsa.prologparser.tokenizer.Op;
13+
import com.igormaznitsa.prologparser.tokenizer.TokenizerResult;
14+
import java.io.StringReader;
15+
import java.util.ArrayList;
16+
import java.util.List;
717
import org.junit.jupiter.api.Test;
818

919
public class PrologParserExceptionTest {
@@ -72,4 +82,32 @@ public void testContainsRightPositionData() {
7282
public void testToString() {
7383
assertEquals("Hello World[1:10]", new PrologParserException("Hello World", 1, 10).toString());
7484
}
85+
86+
@Test
87+
public void testNotificationAboutComments() throws Exception {
88+
final List<TokenizerResult> foundComments = new ArrayList<>();
89+
final List<PrologTerm> foundTerms = new ArrayList<>();
90+
91+
try (final PrologParser parser = new GenericPrologParser(
92+
new StringReader("% Line1\nhello(/*HHH*/world/*End*/).% Ending comment\n[1,2,3]."),
93+
DefaultParserContext.of(
94+
ParserContext.FLAG_BLOCK_COMMENTS | ParserContext.FLAG_COMMENTS_AS_ATOMS, Op.ISO),
95+
(parser1, comment) -> {
96+
foundComments.add(comment);
97+
}
98+
)) {
99+
parser.forEach(foundTerms::add);
100+
}
101+
102+
assertEquals(4, foundComments.size());
103+
assertEquals(" Line1", foundComments.get(0).getResult().getText());
104+
assertEquals("HHH", foundComments.get(1).getResult().getText());
105+
assertEquals("End", foundComments.get(2).getResult().getText());
106+
assertEquals(" Ending comment", foundComments.get(3).getResult().getText());
107+
108+
assertEquals(2, foundTerms.size());
109+
assertEquals("hello(world)", foundTerms.get(0).toString());
110+
assertEquals("[1, 2, 3]", foundTerms.get(1).toString());
111+
112+
}
75113
}

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /