[Python-checkins] python/dist/src/Python import.c, 2.232, 2.233 marshal.c, 1.78, 1.79

Sun Jun 27 12:51:49 EDT 2004

Update of /cvsroot/python/python/dist/src/Python
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv25751/Python
Modified Files:
	import.c marshal.c 
Log Message:
Patch #923098: Share interned strings in marshal.
Index: import.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Python/import.c,v
retrieving revision 2.232
retrieving revision 2.233
diff -C2 -d -r2.232 -r2.233
*** import.c	7 Jun 2004 15:04:10 -0000	2.232
--- import.c	27 Jun 2004 16:51:46 -0000	2.233
***************
*** 27,33 ****
 Apple MPW compiler swaps their values, botching string constants.

! Apparently, there was a distinction made between even and odd
! bytecodes that is related to Unicode. The details aren't clear,
! but the magic number has been odd for a long time.

 There were a variety of old schemes for setting the magic number.
--- 27,33 ----
 Apple MPW compiler swaps their values, botching string constants.

! The magic numbers must be spaced apart atleast 2 values, as the
! -U interpeter flag will cause MAGIC+1 being used. They have been
! odd numbers for some time now.

 There were a variety of old schemes for setting the magic number.
***************
*** 48,54 ****
 Python 2.3a0: 62021
 Python 2.3a0: 62011 (!)
! Python 2.4a0: 62031
 */
! #define MAGIC (62031 | ((long)'\r'<<16) | ((long)'\n'<<24))

 /* Magic word as global; note that _PyImport_Init() can change the
--- 48,54 ----
 Python 2.3a0: 62021
 Python 2.3a0: 62011 (!)
! Python 2.4a0: 62041
 */
! #define MAGIC (62041 | ((long)'\r'<<16) | ((long)'\n'<<24))

 /* Magic word as global; note that _PyImport_Init() can change the
***************
*** 798,805 ****
 		return;
 	}
! 	PyMarshal_WriteLongToFile(pyc_magic, fp);
 	/* First write a 0 for mtime */
! 	PyMarshal_WriteLongToFile(0L, fp);
! 	PyMarshal_WriteObjectToFile((PyObject *)co, fp);
 	if (fflush(fp) != 0 || ferror(fp)) {
 		if (Py_VerboseFlag)
--- 798,805 ----
 		return;
 	}
! 	PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION);
 	/* First write a 0 for mtime */
! 	PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION);
! 	PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION);
 	if (fflush(fp) != 0 || ferror(fp)) {
 		if (Py_VerboseFlag)
***************
*** 812,816 ****
 	/* Now write the true mtime */
 	fseek(fp, 4L, 0);
! 	PyMarshal_WriteLongToFile(mtime, fp);
 	fflush(fp);
 	fclose(fp);
--- 812,816 ----
 	/* Now write the true mtime */
 	fseek(fp, 4L, 0);
! 	PyMarshal_WriteLongToFile(mtime, fp, Py_MARSHAL_VERSION);
 	fflush(fp);
 	fclose(fp);
Index: marshal.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Python/marshal.c,v
retrieving revision 1.78
retrieving revision 1.79
diff -C2 -d -r1.78 -r1.79
*** marshal.c	13 Jun 2004 20:31:49 -0000	1.78
--- marshal.c	27 Jun 2004 16:51:46 -0000	1.79
***************
*** 28,31 ****
--- 28,33 ----
 #define TYPE_LONG	'l'
 #define TYPE_STRING	's'
+ #define TYPE_INTERNED	't'
+ #define TYPE_STRINGREF	'R'
 #define TYPE_TUPLE	'('
 #define TYPE_LIST	'['
***************
*** 43,46 ****
--- 45,49 ----
 	char *ptr;
 	char *end;
+ 	PyObject *strings; /* dict on marshal, list on unmarshal */
 } WFILE;

***************
*** 190,194 ****
 #endif
 	else if (PyString_Check(v)) {
! 		w_byte(TYPE_STRING, p);
 		n = PyString_GET_SIZE(v);
 		w_long((long)n, p);
--- 193,214 ----
 #endif
 	else if (PyString_Check(v)) {
! 		if (p->strings && PyString_CHECK_INTERNED(v)) {
! 			PyObject *o = PyDict_GetItem(p->strings, v);
! 			if (o) {
! 				long w = PyInt_AsLong(o);
! 				w_byte(TYPE_STRINGREF, p);
! 				w_long(w, p);
! 				goto exit;
! 			}
! 			else {
! 				o = PyInt_FromLong(PyDict_Size(p->strings));
! 				PyDict_SetItem(p->strings, v, o);
! 				Py_DECREF(o);
! 				w_byte(TYPE_INTERNED, p);
! 			}
! 		}
! 		else {
! 			w_byte(TYPE_STRING, p);
! 		}
 		n = PyString_GET_SIZE(v);
 		w_long((long)n, p);
***************
*** 270,279 ****
 		p->error = 1;
 	}
! 
 	p->depth--;
 }

 void
! PyMarshal_WriteLongToFile(long x, FILE *fp)
 {
 	WFILE wf;
--- 290,300 ----
 		p->error = 1;
 	}
! exit:
 	p->depth--;
 }

+ /* version currently has no effect for writing longs. */
 void
! PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
 {
 	WFILE wf;
***************
*** 281,289 ****
 	wf.error = 0;
 	wf.depth = 0;
 	w_long(x, &wf);
 }

 void
! PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp)
 {
 	WFILE wf;
--- 302,311 ----
 	wf.error = 0;
 	wf.depth = 0;
+ 	wf.strings = NULL;
 	w_long(x, &wf);
 }

 void
! PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
 {
 	WFILE wf;
***************
*** 291,295 ****
--- 313,319 ----
 	wf.error = 0;
 	wf.depth = 0;
+ 	wf.strings = (version > 0) ? PyDict_New() : NULL;
 	w_object(x, &wf);
+ 	Py_XDECREF(wf.strings);
 }

***************
*** 492,495 ****
--- 516,520 ----
 #endif

+ 	case TYPE_INTERNED:
 	case TYPE_STRING:
 		n = r_long(p);
***************
*** 507,510 ****
--- 532,545 ----
 			}
 		}
+ 		if (type == TYPE_INTERNED) {
+ 			PyString_InternInPlace(&v);
+ 			PyList_Append(p->strings, v);
+ 		}
+ 		return v;
+ 
+ 	case TYPE_STRINGREF:
+ 		n = r_long(p);
+ 		v = PyList_GET_ITEM(p->strings, n);
+ 		Py_INCREF(v);
 		return v;

***************
*** 674,677 ****
--- 709,713 ----
 	RFILE rf;
 	rf.fp = fp;
+ 	rf.strings = NULL;
 	return r_short(&rf);
 }
***************
*** 682,685 ****
--- 718,722 ----
 	RFILE rf;
 	rf.fp = fp;
+ 	rf.strings = NULL;
 	return r_long(&rf);
 }
***************
*** 748,753 ****
 {
 	RFILE rf;
 	rf.fp = fp;
! 	return read_object(&rf);
 }

--- 785,794 ----
 {
 	RFILE rf;
+ 	PyObject *result;
 	rf.fp = fp;
! 	rf.strings = PyList_New(0);
! 	result = r_object(&rf);
! 	Py_DECREF(rf.strings);
! 	return result;
 }

***************
*** 756,767 ****
 {
 	RFILE rf;
 	rf.fp = NULL;
 	rf.ptr = str;
 	rf.end = str + len;
! 	return read_object(&rf);
 }

 PyObject *
! PyMarshal_WriteObjectToString(PyObject *x) /* wrs_object() */
 {
 	WFILE wf;
--- 797,812 ----
 {
 	RFILE rf;
+ 	PyObject *result;
 	rf.fp = NULL;
 	rf.ptr = str;
 	rf.end = str + len;
! 	rf.strings = PyList_New(0);
! 	result = r_object(&rf);
! 	Py_DECREF(rf.strings);
! 	return result;
 }

 PyObject *
! PyMarshal_WriteObjectToString(PyObject *x, int version)
 {
 	WFILE wf;
***************
*** 774,778 ****
--- 819,825 ----
 	wf.error = 0;
 	wf.depth = 0;
+ 	wf.strings = (version > 0) ? PyDict_New() : NULL;
 	w_object(x, &wf);
+ 	Py_XDECREF(wf.strings);
 	if (wf.str != NULL)
 		_PyString_Resize(&wf.str,
***************
*** 797,801 ****
 	PyObject *x;
 	PyObject *f;
! 	if (!PyArg_ParseTuple(args, "OO:dump", &x, &f))
 		return NULL;
 	if (!PyFile_Check(f)) {
--- 844,849 ----
 	PyObject *x;
 	PyObject *f;
! 	int version = Py_MARSHAL_VERSION;
! 	if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version))
 		return NULL;
 	if (!PyFile_Check(f)) {
***************
*** 809,813 ****
--- 857,863 ----
 	wf.error = 0;
 	wf.depth = 0;
+ 	wf.strings = (version > 0) ? PyDict_New() : 0;
 	w_object(x, &wf);
+ 	Py_XDECREF(wf.strings);
 	if (wf.error) {
 		PyErr_SetString(PyExc_ValueError,
***************
*** 824,828 ****
 {
 	RFILE rf;
! 	PyObject *f;
 	if (!PyArg_ParseTuple(args, "O:load", &f))
 		return NULL;
--- 874,878 ----
 {
 	RFILE rf;
! 	PyObject *f, *result;
 	if (!PyArg_ParseTuple(args, "O:load", &f))
 		return NULL;
***************
*** 833,837 ****
 	}
 	rf.fp = PyFile_AsFile(f);
! 	return read_object(&rf);
 }

--- 883,890 ----
 	}
 	rf.fp = PyFile_AsFile(f);
! 	rf.strings = PyList_New(0);
! 	result = read_object(&rf);
! 	Py_DECREF(rf.strings);
! 	return result;
 }

***************
*** 840,846 ****
 {
 	PyObject *x;
! 	if (!PyArg_ParseTuple(args, "O:dumps", &x))
 		return NULL;
! 	return PyMarshal_WriteObjectToString(x);
 }

--- 893,900 ----
 {
 	PyObject *x;
! 	int version = Py_MARSHAL_VERSION;
! 	if (!PyArg_ParseTuple(args, "O|i:dumps", &x, version))
 		return NULL;
! 	return PyMarshal_WriteObjectToString(x, version);
 }

***************
*** 851,860 ****
 	char *s;
 	int n;
! 	if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
 		return NULL;
 	rf.fp = NULL;
 	rf.ptr = s;
 	rf.end = s + n;
! 	return read_object(&rf);
 }

--- 905,918 ----
 	char *s;
 	int n;
! 	PyObject* result;
! 	if (!PyArg_ParseTuple(args, "s#|i:loads", &s, &n))
 		return NULL;
 	rf.fp = NULL;
 	rf.ptr = s;
 	rf.end = s + n;
! 	rf.strings = PyList_New(0);
! 	result = read_object(&rf);
! 	Py_DECREF(rf.strings);
! 	return result;
 }

***************
*** 870,873 ****
 PyMarshal_Init(void)
 {
! 	(void) Py_InitModule("marshal", marshal_methods);
 }
--- 928,932 ----
 PyMarshal_Init(void)
 {
! 	PyObject *mod = Py_InitModule("marshal", marshal_methods);
! 	PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
 }