Return to Question

1. Parsing the .bc3 file uploaded by the user.

Everything is working as expected.

2. Instantiating the Concept model.

I save the instances in concept_instances = [c1, c2, c3... cn].

3. Inserting Concept instances into the database.

In order to speed up the load I use the bulk_create(concept_instances) method.

4. Instantiating the Deco model.

I save the instances in deco_instances = [d1, d2, d3... dn]. But, to do that I need to retrieve eachConcept object from the database because of the parent_concept and concept fields.

5. Inserting Deco instances into the database.

As before, to speed up the load I use the bulk_create(deco_instances) method.

1. Parsing the .bc3 file uploaded by the user.

   Everything is working as expected.

2. Instantiating the Concept model.

   I save the instances in concept_instances = [c1, c2, c3... cn].

3. Inserting Concept instances into the database.

   In order to speed up the load I use the bulk_create(concept_instances) method.

4. Instantiating the Deco model.

   I save the instances in deco_instances = [d1, d2, d3... dn]. But, to do that I need to retrieve eachConcept object from the database because of the parent_concept and concept fields.

5. Inserting Deco instances into the database.

   As before, to speed up the load I use the bulk_create(deco_instances) method.

class Concept(models.Model):
 code = models.CharField(_('code'), max_length=20, unique=True)
 root = models.BooleanField(_('is it root'), default=False)
 chapter = models.BooleanField(_('is it chapter'), default=False)
 parent = models.BooleanField(_('is it parent'), default=False)
 unit = models.CharField(_('unit'), blank=True, max_length=3)
 summary = models.CharField(_('summary'), blank=True, max_length=100)
 price = models.DecimalField(_('price'), max_digits=12, decimal_places=3,
 null=True, blank=True)
 date = models.DateField(_('creation date'), null=True, blank=True)
 concept_type = models.CharField(_('concept type'), max_length=3, blank=True)
 def __str__(self):
 return '%s: %s' % (self.code, self.summary)
class Deco(models.Model):
 parent_concept = models.ForeignKey(Concept, null=True, blank=True,
 related_name='decos')
 concept = models.ForeignKey(Concept, null=True, blank=True)
 factor = models.DecimalField(max_digits=12, decimal_places=3,
 default=Decimal('0.000'))
 efficiency = models.DecimalField(max_digits=12, decimal_places=3,
 default=Decimal('0.000'))
 def __str__(self):
 return '%s: %s' % (self.parent_concept, self.concept)

class Concept(models.Model):
 code = models.CharField(_('code'), max_length=20, primary_key=True)
 root = models.BooleanField(_('is it root'), default=False)
 chapter = models.BooleanField(_('is it chapter'), default=False)
 parent = models.BooleanField(_('is it parent'), default=False)
 unit = models.CharField(_('unit'), blank=True, max_length=3)
 summary = models.CharField(_('summary'), blank=True, max_length=100)
 price = models.DecimalField(_('price'), max_digits=12, decimal_places=3,
 null=True, blank=True)
 date = models.DateField(_('creation date'), null=True, blank=True)
 concept_type = models.CharField(_('concept type'), max_length=3, blank=True)
 def __str__(self):
 return '%s: %s' % (self.code, self.summary)
class Deco(models.Model):
 parent_concept = models.ForeignKey(Concept, null=True, blank=True,
 related_name='decos')
 concept = models.ForeignKey(Concept, null=True, blank=True)
 factor = models.DecimalField(max_digits=12, decimal_places=3,
 default=Decimal('0.000'))
 efficiency = models.DecimalField(max_digits=12, decimal_places=3,
 default=Decimal('0.000'))
 def __str__(self):
 return '%s: %s' % (self.parent_concept, self.concept)

Process

bc3parser.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Parses bc3 files and insert all the data into the database."""
import re
from enkendas.models import Version, Concept, Deco, Text
from .utils import optional_codes, parse_dates
# regex stuff
# parsers stuff
concepts = {}
decos = {}
# decos = {'PER02': [('Qexcav', '1', '231.13'), ('Qzanj', '1', '34.5'),
# ('Qexcav2', '1', '19.07'), ('Qrelltras', '1', '19.07')],
# ...
# 'Qexcav': [('MMMT.3c', '1', '0.045'), ('O01OA070', '1', '0.054'),
# ('M07CB030', '1', '0.036'), ('%0300', '1', '0.03')]}
def dispatch_record(record):
 """
 Dispatch every record.
 Check the first character of the record and send it to the proper function.
 """
 if record.startswith('D'):
 parse_decomp(record)
 elif record.startswith('V'):
 parse_version(record)
 elif record.startswith('C'):
 parse_concept(record)
 elif record.startswith('T'):
 parse_text(record)
 else:
 pass
def parse_file(file):
 """
 Parse the whole file.
 file is a generator returned by file.chunks(chunk_size=80000) in views.py.
 """
 while True:
 try:
 record = ''
 incomplete_record = ''
 # Iterates over the file sent by the user.
 byte_string = next(file)
 byte_stripped_string = byte_string.strip()
 string = byte_stripped_string.decode(encoding='ISO-8859-1')
 # List of records.
 durty_strings_list = string.split('~')
 # Check if one chunk in chunks is complete.
 if durty_strings_list[-1] != '' and incomplete_record != '':
 incomplete_record = incomplete_record + durty_strings_list.pop(-1)
 dispatch_record(incomplete_record)
 incomplete_record = ''
 elif durty_strings_list[-1] != '' and incomplete_record == '':
 incomplete_record = durty_strings_list.pop(-1)
 for durty_string in durty_strings_list:
 stripped_string = durty_string.strip()
 if durty_string == '':
 record = record + ''
 # TODO: I didn't create a regex for 'M' and 'E' records yet.
 elif durty_string[0] == 'M' or durty_string[0] == 'E':
 continue
 if record != '':
 # Dispatch the previous record.
 dispatch_record(record)
 # Reset the used record.
 record = ''
 # Assign the current record.
 record = stripped_string
 else:
 record = record + stripped_string
 except StopIteration as e:
 dispatch_record(record)
 break
 concept_instances = []
 for key_code, data in concepts.items():
 code = key_code
 root = chapter = parent = False
 if len(key_code) > 2 and key_code[-2:] == '##':
 root = True
 code = key_code[:-2]
 elif len(key_code) > 1 and key_code[-1:] == '#':
 chapter = True
 code = key_code[:-1]
 if code in decos:
 parent = True
 concept = Concept(code=code, root=root, chapter=chapter, parent=parent,
 unit=data['unit'], summary=data['summary'],
 price=data['price'], date=data['date'],
 concept_type=data['concept_type'])
 concept_instances.append(concept)
 Concept.objects.bulk_create(concept_instances)
 deco_instances = []
 cobjs_storage = {}
 for concept in Concept.objects.all():
 if concept.parent is False:
 continue
 dec = decos[concept.code]
 for child, factor, efficiency in dec:
 if child == '':
 continue
 if factor == '':
 factor = '0.000'
 if efficiency == '':
 efficiency = '0.000'
 # To avoid extra queries.
 if child in cobjs_storage:
 cobj = cobjs_storage[child]
 else:
 cobj = Concept.objects.get(code=child)
 cobjs_storage.update({child: cobj})
 deco = Deco(parent_concept=concept, concept=cobj,
 factor=float(factor), efficiency=float(efficiency))
 deco_instances.append(deco)
 decos.pop(concept.code, None)
 Deco.objects.bulk_create(deco_instances)

Process