Commit 620ac3b

committed

Updated corenlp_protobuf to include writing functions

1 parent 99340a9 commit 620ac3bCopy full SHA for 620ac3b

File tree

+27

-4

lines changed

+27

-4

lines changed

Lines changed: 17 additions & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -1,9 +1,11 @@`
`1`	`1`	`from __future__ import absolute_import`
`2`	`2`
	`3`	`+from io import BytesIO`
	`4`	`+`
	`5`	`+from google.protobuf.internal.encoder import _EncodeVarint`
`3`	`6`	`from google.protobuf.internal.decoder import _DecodeVarint`
`4`	`7`	`from .CoreNLP_pb2 import *`
`5`	`8`
`6`		`-`
`7`	`9`	`def parseFromDelimitedString(obj, buf, offset=0):`
`8`	`10`	`"""`
`9`	`11`	`Stanford CoreNLP uses the Java "writeDelimitedTo" function, which`
`@@ -16,6 +18,20 @@ def parseFromDelimitedString(obj, buf, offset=0):`
`16`	`18`	`obj.ParseFromString(buf[offset+pos:offset+pos+size])`
`17`	`19`	`return pos+size`
`18`	`20`
	`21`	`+def writeToDelimitedString(obj, stream=None):`
	`22`	`+ """`
	`23`	`+ Stanford CoreNLP uses the Java "writeDelimitedTo" function, which`
	`24`	`+ writes the size (and offset) of the buffer before writing the object.`
	`25`	`+ This function handles parsing this message starting from offset 0.`
	`26`	`+`
	`27`	`+ @returns how many bytes of @buf were consumed.`
	`28`	`+ """`
	`29`	`+ if stream is None:`
	`30`	`+ stream = BytesIO()`
	`31`	`+`
	`32`	`+ _EncodeVarint(stream.write, obj.ByteSize())`
	`33`	`+ stream.write(obj.SerializeToString())`
	`34`	`+ return stream`
`19`	`35`
`20`	`36`	`def to_text(sentence):`
`21`	`37`	`"""`

Lines changed: 1 addition & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -23,7 +23,7 @@`
`23`	`23`	`# Versions should comply with PEP440. For a discussion on single-sourcing`
`24`	`24`	`# the version across setup.py and the project code, see`
`25`	`25`	`# https://packaging.python.org/en/latest/single_source_version.html`
`26`		`- version='3.7.0',`
	`26`	`+ version='3.7.1',`
`27`	`27`
`28`	`28`	`description='Python bindings for Stanford CoreNLP protobufs',`
`29`	`29`	`long_description=long_description,`

Lines changed: 9 additions & 2 deletions

Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@`
`11`	`11`	`from pytest import fixture`
`12`	`12`	`from corenlp_protobuf import Document, Sentence, Token, DependencyGraph,\`
`13`	`13`	`CorefChain`
`14`		`-from corenlp_protobuf import parseFromDelimitedString, to_text`
	`14`	`+from corenlp_protobuf import parseFromDelimitedString, writeToDelimitedString, to_text`
`15`	`15`
`16`	`16`
`17`	`17`	`# Thext that was annotated`
`@@ -28,10 +28,17 @@ def doc_pb():`
`28`	`28`	`parseFromDelimitedString(doc, buf)`
`29`	`29`	`return doc`
`30`	`30`
`31`		`-`
`32`	`31`	`def test_parse_protobuf(doc_pb):`
`33`	`32`	`assert doc_pb.ByteSize() == 4239`
`34`	`33`
	`34`	`+def test_write_protobuf(doc_pb):`
	`35`	`+ stream = writeToDelimitedString(doc_pb)`
	`36`	`+ buf = stream.getvalue()`
	`37`	`+ stream.close()`
	`38`	`+`
	`39`	`+ doc_pb_ = Document()`
	`40`	`+ parseFromDelimitedString(doc_pb_, buf)`
	`41`	`+ assert doc_pb == doc_pb_`
`35`	`42`
`36`	`43`	`def test_document_text(doc_pb):`
`37`	`44`	`assert doc_pb.text == TEXT`

Comments

(0)