using pandas and numpy to parse dirty .csv

Question

I'm relatively new to Python have been writing for a few months now. I've started a Pandas Numpy project that starts with parsing large and somewhat sloppy formatted textfile, its not exactly csv but its pretty close.

...
Basetime: 2021102206Z
Forecast Hours: 0hr 1hr... 144hr 
Sfc Prs(mb): 1002.83, 1002.62,...
...
Dry Microburst: 0.00, 0,...
###UA SECTION###
1000mb GPH (m): 166.88,... 
...
#END LINE 861

create instance from file

• pd.DataFrame
• numpy.ndarray

with open('data/2021102206Z.txt', 'r') as f:
 tp = Tarp(f)
 df = tp.asDataFrame()
 print(type(df))
 #<class 'pandas.core.frame.DataFrame'>
 m = tp.asMatrix()
 print(type(m))
 #<class 'numpy.ndarray'>

init

Tarp class parses the applicable data into a Object subclass

list(map(self._series, f)) parse -> reject -> reformat -> setatt

class Tarp(object):
 def __init__(self, f):
 self.props = Object()
 self.models = Object()
 # itterate f as pd.Series
 
 def _series(self, f):
 # split series at : or , or whitespace
 s = pd.Series(f).str.split(r":\s*|,\s*").squeeze()
 # pop key from series to reformat, returns False if key invalid
 k = _pythonic_keys(s.pop(0))
 v = _pythonic_vals(s)
 try:
 if k and v:# k & v = model data
 # force ValueError on non dtype float
 val = np.array(v, dtype=float)
 self.models.keys.append(k)
 setattr(self.models, k, val)
 pass
 elif k:# k not v = station properties
 v = s[0].rstrip()
 if k == "basetime":
 val = datetime.strptime(v, "%Y%m%d%HZ")
 else:
 val = v
 self.props.keys.append(k)
 setattr(self.props, k, val)
 except ValueError:# occurs on int array vals
 pass
 except IndexError:# occurs on blank lines
 pass

Methods

 def getmodels(self, key): return getattr(self.models, key)
 def getprops(self, key): return getattr(self.props, key)
 def zip_props(self):
 vals = list(map(self.getprops, self.props.keys))
 return dict(zip(self.props.keys, vals))
 def zip_models(self):
 vals = list(map(self.getmodels, self.models.keys))
 return dict(zip(self.models.keys, vals))
 def asMatrix(self): return np.array(
 list(map(self.getmodels, self.models.keys)))
 def asDataFrame(self): return pd.DataFrame(self.zip_models())

misc

class Object:
 def __init__(self):
 self.keys = list()
 pass
_wspace = re.compile(r'\s+')
_units = re.compile(r'(?=\s*\(.*)')
_only_wspace = re.compile(r"^\s*$")
def _pythonic_keys(s):
 if bool(_only_wspace.search(s)):
 return False
 else:
 return _wspace.sub("_", _units.split(s)[0]).lower()
def _pythonic_vals(a):
 if len(a) > 0:
 return a[:-1]
 else:
 return False

full code

import numpy as np
import pandas as pd
import re
from datetime import datetime
_wspace = re.compile(r'\s+')
_units = re.compile(r'(?=\s*\(.*)')
_only_wspace = re.compile(r"^\s*$")
def _pythonic_keys(s):
 if bool(_only_wspace.search(s)):
 return False
 else:
 return _wspace.sub("_", _units.split(s)[0]).lower()
def _pythonic_vals(a):
 if len(a) > 0:
 return a[:-1]
 else:
 return False
class Object:
 def __init__(self):
 self.keys = list()
 pass
class Tarp(object):
 def getmodels(self, key): return getattr(self.models, key)
 def getprops(self, key): return getattr(self.props, key)
 def zip_props(self):
 vals = list(map(self.getprops, self.props.keys))
 return dict(zip(self.props.keys, vals))
 def zip_models(self):
 vals = list(map(self.getmodels, self.models.keys))
 return dict(zip(self.models.keys, vals))
 def asMatrix(self): return np.array(
 list(map(self.getmodels, self.models.keys)))
 def asDataFrame(self): return pd.DataFrame(self.zip_models())
 def __init__(self, f):
 self.props = Object()
 self.models = Object()
 # itterate f as pd.Series
 list(map(self._series, f))
 def _series(self, f):
 # split series at : or , or whitespace
 s = pd.Series(f).str.split(r":\s*|,\s*").squeeze()
 # pop key from series to reformat, returns False if key invalid
 k = _pythonic_keys(s.pop(0))
 v = _pythonic_vals(s)
 try:
 if k and v:
 # force ValueError on non dtype float
 val = np.array(v, dtype=float)
 self.models.keys.append(k)
 setattr(self.models, k, val)
 pass
 elif k:
 v = s[0].rstrip()
 if k == "basetime":
 val = datetime.strptime(v, "%Y%m%d%HZ")
 else:
 val = v
 self.props.keys.append(k)
 setattr(self.props, k, val)
 except ValueError:
 pass
 except IndexError:
 pass
def run_test():
 with open('data/2021102206Z.txt', 'r') as f:
 tp = Tarp(f)
 df = tp.asDataFrame()
 print(type(df))
 m = tp.asMatrix()
 print(type(m))

more data

the file is a 144hr forecast by hour for several different values

at line 160 is there a break and the values are upper level values

 
Basetime: 2021102206Z
Spacing: 1/4 Degree 
Region: Worldwide 
StationID: TEST
Station Name: TEST DATA
Latitude: 50.55
Longitude: -100.84
Elevation: 139.9
Forecast Hours: 0hr 1hr 2hr 3hr 4hr 5hr 6hr 7hr 8hr 9hr 10hr 11hr 12hr 13hr 14hr 15hr 16hr 17hr 18hr 19hr 20hr 21hr 22hr 23hr 24hr 25hr 26hr 27hr 28hr 29hr 30hr 31hr 32hr 33hr 34hr 35hr 36hr 37hr 38hr 39hr 40hr 41hr 42hr 43hr 44hr 45hr 46hr 47hr 48hr 49hr 50hr 51hr 52hr 53hr 54hr 55hr 56hr 57hr 58hr 59hr 60hr 61hr 62hr 63hr 64hr 65hr 66hr 67hr 68hr 69hr 70hr 71hr 72hr 73hr 74hr 75hr 76hr 77hr 78hr 79hr 80hr 81hr 82hr 83hr 84hr 85hr 86hr 87hr 88hr 89hr 90hr 91hr 92hr 93hr 94hr 95hr 96hr 97hr 98hr 99hr 100hr 101hr 102hr 103hr 104hr 105hr 106hr 107hr 108hr 109hr 110hr 111hr 112hr 113hr 114hr 115hr 116hr 117hr 118hr 119hr 120hr 121hr 122hr 123hr 124hr 125hr 126hr 127hr 128hr 129hr 130hr 131hr 132hr 133hr 134hr 135hr 136hr 137hr 138hr 139hr 140hr 141hr 142hr 143hr 144hr
Sfc Prs(mb): 1002.83, 1002.62, 1002.33, 1001.96, 1002.13, 1002.10, 1002.16, 1002.59, 1002.68, 1002.58, 1002.44, 1002.31, 1001.81, 1001.26, 1000.83, 1000.46, 1000.30, 1000.26, 1000.34, 1000.17, 1000.31, 1000.38, 1000.18, 1000.07, 1000.18, 1000.17, 1000.01, 999.85, 999.90, 1000.10, 1000.22, 1000.38, 1000.66, 1000.65, 1000.32, 999.84, 999.16, 998.25, 997.31, 996.85, 996.56, 996.46, 996.74, 996.60, 996.46, 996.32, 995.94, 995.55, 995.17, 994.58, 994.00, 993.41, 993.15, 992.90, 992.64, 992.88, 993.11, 993.34, 992.44, 991.53, 990.63, 989.95, 989.27, 988.59, 988.02, 987.45, 986.88, 986.61, 986.33, 986.06, 985.86, 985.67, 985.48, 985.51, 985.54, 985.56, 986.08, 986.60, 987.12, 987.67, 988.23, 988.79, 988.84, 988.89, 988.95, 989.69, 990.43, 991.18, 992.38, 993.59, 994.79, 995.47, 996.14, 996.81, 997.15, 997.50, 997.84, 997.99, 998.14, 998.30, 998.73, 999.17, 999.61, 999.81, 1000.01, 1000.22, 999.47, 998.73, 997.99, 997.28, 996.57, 995.86, 995.74, 995.63, 995.52, 994.82, 994.11, 993.41, 992.59, 991.77, 990.95, 990.60, 990.26, 989.91, 988.19, 986.47, 984.76, 983.96, 983.17, 982.37, 981.11, 979.84, 978.57, 978.14, 977.71, 977.28, 977.57, 977.85, 978.13, 978.46, 978.79, 979.12, 979.30, 979.49, 979.68,
Mean SLP (mb): 1019.73, 1019.50, 1019.19, 1018.83, 1019.03, 1019.02, 1019.09, 1019.54, 1019.60, 1019.49, 1019.34, 1019.21, 1018.73, 1018.18, 1017.73, 1017.35, 1017.21, 1017.19, 1017.27, 1017.07, 1017.23, 1017.32, 1017.14, 1017.05, 1017.20, 1017.17, 1016.98, 1016.80, 1016.77, 1016.92, 1017.03, 1017.19, 1017.46, 1017.49, 1017.15, 1016.69, 1016.00, 1015.07, 1014.09, 1013.56, 1013.25, 1013.14, 1013.41, 1013.26, 1013.12, 1012.97, 1012.56, 1012.16, 1011.76, 1011.15, 1010.54, 1009.92, 1009.64, 1009.35, 1009.06, 1009.29, 1009.51, 1009.73, 1008.81, 1007.88, 1006.96, 1006.29, 1005.62, 1004.95, 1004.36, 1003.76, 1003.16, 1002.92, 1002.68, 1002.44, 1002.27, 1002.10, 1001.93, 1001.99, 1002.04, 1002.09, 1002.67, 1003.24, 1003.81, 1004.37, 1004.92, 1005.48, 1005.50, 1005.53, 1005.55, 1006.30, 1007.06, 1007.81, 1009.03, 1010.25, 1011.47, 1012.14, 1012.81, 1013.48, 1013.81, 1014.15, 1014.49, 1014.61, 1014.74, 1014.87, 1015.31, 1015.75, 1016.19, 1016.39, 1016.60, 1016.81, 1016.05, 1015.28, 1014.52, 1013.80, 1013.08, 1012.35, 1012.23, 1012.12, 1012.00, 1011.26, 1010.51, 1009.77, 1008.95, 1008.13, 1007.31, 1006.99, 1006.68, 1006.36, 1004.63, 1002.89, 1001.16, 1000.39, 999.61, 998.84, 997.59, 996.35, 995.10, 994.70, 994.30, 993.90, 994.18, 994.47, 994.76, 995.10, 995.44, 995.79, 995.99, 996.20, 996.41,
Altimeter (in. Hg): 30.10, 30.09, 30.09, 30.07, 30.08, 30.08, 30.08, 30.09, 30.10, 30.09, 30.09, 30.08, 30.07, 30.05, 30.04, 30.03, 30.02, 30.02, 30.03, 30.02, 30.02, 30.03, 30.02, 30.02, 30.02, 30.02, 30.02, 30.01, 30.01, 30.02, 30.02, 30.03, 30.04, 30.04, 30.03, 30.01, 29.99, 29.96, 29.93, 29.92, 29.91, 29.91, 29.92, 29.91, 29.91, 29.91, 29.89, 29.88, 29.87, 29.85, 29.84, 29.82, 29.81, 29.80, 29.80, 29.80, 29.81, 29.82, 29.79, 29.76, 29.74, 29.71, 29.69, 29.67, 29.66, 29.64, 29.62, 29.61, 29.61, 29.60, 29.59, 29.59, 29.58, 29.58, 29.58, 29.58, 29.60, 29.61, 29.63, 29.65, 29.66, 29.68, 29.68, 29.68, 29.68, 29.71, 29.73, 29.75, 29.79, 29.82, 29.86, 29.88, 29.90, 29.92, 29.93, 29.94, 29.95, 29.96, 29.96, 29.96, 29.98, 29.99, 30.00, 30.01, 30.02, 30.02, 30.00, 29.98, 29.96, 29.93, 29.91, 29.89, 29.89, 29.88, 29.88, 29.86, 29.84, 29.82, 29.79, 29.77, 29.74, 29.73, 29.72, 29.71, 29.66, 29.61, 29.56, 29.54, 29.51, 29.49, 29.45, 29.41, 29.37, 29.36, 29.35, 29.34, 29.34, 29.35, 29.36, 29.37, 29.38, 29.39, 29.40, 29.40, 29.41,
Press Alt (ft): 285.78, 291.39, 299.50, 309.49, 304.92, 305.71, 304.16, 292.24, 289.64, 292.43, 296.33, 300.04, 313.61, 328.87, 340.80, 350.93, 355.33, 356.43, 354.22, 359.08, 355.09, 353.16, 358.68, 361.81, 358.63, 358.82, 363.37, 367.77, 366.37, 360.80, 357.50, 353.13, 345.54, 345.62, 354.75, 367.99, 386.75, 411.88, 438.08, 450.74, 458.64, 461.38, 453.69, 457.56, 461.43, 465.29, 475.92, 486.54, 497.17, 513.43, 529.69, 545.96, 553.06, 560.16, 567.26, 560.77, 554.29, 547.81, 572.92, 598.05, 623.21, 642.09, 660.98, 679.89, 695.76, 711.65, 727.54, 735.22, 742.90, 750.58, 755.95, 761.32, 766.69, 765.90, 765.11, 764.31, 749.87, 735.44, 721.01, 705.45, 689.90, 674.36, 672.92, 671.48, 670.04, 649.34, 628.65, 607.97, 574.49, 541.04, 507.62, 489.00, 470.38, 451.77, 442.24, 432.72, 423.20, 419.04, 414.88, 410.72, 398.59, 386.46, 374.33, 368.79, 363.24, 357.69, 378.18, 398.68, 419.19, 438.83, 458.49, 478.17, 481.28, 484.40, 487.51, 506.98, 526.46, 545.95, 568.71, 591.48, 614.27, 623.91, 633.55, 643.19, 691.02, 738.91, 786.87, 809.08, 831.29, 853.53, 888.99, 924.49, 960.03, 972.11, 984.19, 996.27, 988.33, 980.39, 972.45, 963.21, 953.97, 944.74, 939.51, 934.28, 929.05,
Density Alt (ft): -55.22, -88.18, -135.38, -186.09, -235.12, -262.16, -312.89, -296.80, -142.70, -0.45, 88.61, 172.51, 274.47, 348.90, 398.48, 378.66, 356.46, 249.88, 90.22, -10.37, -101.92, -182.53, -205.18, -231.41, -338.80, -394.94, -419.42, -427.96, -418.72, -433.90, -436.02, -374.09, -103.15, 200.48, 426.54, 563.89, 643.07, 731.70, 768.59, 851.75, 836.91, 774.23, 678.68, 684.24, 689.77, 695.11, 705.35, 715.59, 725.83, 742.34, 758.67, 775.21, 850.91, 926.88, 1003.14, 1132.49, 1262.30, 1392.82, 1523.68, 1654.20, 1784.81, 1877.47, 1969.33, 2060.37, 2037.23, 2014.25, 1991.25, 1945.64, 1899.72, 1853.71, 1794.06, 1734.05, 1673.69, 1599.03, 1524.64, 1450.48, 1272.28, 1095.99, 921.63, 829.19, 736.56, 643.31, 710.27, 777.51, 845.04, 795.05, 744.84, 694.83, 596.17, 497.41, 398.98, 341.93, 284.85, 227.93, 189.31, 150.86, 112.19, 91.00, 70.00, 48.99, 26.97, 4.95, -16.86, 24.73, 66.02, 107.24, 197.02, 286.22, 374.83, 445.25, 515.60, 585.66, 529.25, 472.31, 415.02, 406.81, 398.58, 390.32, 383.68, 377.02, 370.56, 363.20, 355.83, 348.45, 419.28, 490.17, 561.13, 643.70, 726.36, 808.92, 948.00, 1087.20, 1226.51, 1339.39, 1451.95, 1564.38, 1478.83, 1393.31, 1307.80, 1265.79, 1224.16, 1182.28, 1137.12, 1091.92, 1046.88,
2 m agl Tmp (K): 283.50, 283.17, 282.70, 282.18, 281.82, 281.59, 281.20, 281.43, 282.70, 283.86, 284.59, 285.24, 285.94, 286.41, 286.71, 286.48, 286.27, 285.40, 284.10, 283.22, 282.52, 281.89, 281.62, 281.34, 280.52, 280.08, 279.82, 279.69, 279.76, 279.70, 279.71, 280.24, 282.49, 284.96, 286.70, 287.66, 288.10, 288.54, 288.53, 289.02, 288.78, 288.20, 287.45, 287.42, 287.39, 287.36, 287.35, 287.33, 287.32, 287.31, 287.29, 287.28, 287.77, 288.26, 288.75, 289.78, 290.81, 291.85, 292.62, 293.40, 294.18, 294.80, 295.41, 296.03, 295.72, 295.42, 295.12, 294.66, 294.20, 293.74, 293.18, 292.61, 292.05, 291.51, 290.97, 290.44, 289.30, 288.16, 287.02, 286.41, 285.80, 285.19, 285.68, 286.16, 286.64, 286.45, 286.27, 286.08, 285.64, 285.21, 284.77, 284.53, 284.29, 284.05, 283.83, 283.61, 283.39, 283.26, 283.12, 282.99, 282.92, 282.86, 282.80, 283.19, 283.59, 283.99, 284.55, 285.12, 285.68, 286.07, 286.45, 286.84, 286.33, 285.81, 285.30, 285.01, 284.72, 284.44, 284.16, 283.88, 283.60, 283.42, 283.24, 283.06, 283.12, 283.19, 283.25, 283.67, 284.09, 284.51, 285.23, 285.95, 286.68, 287.46, 288.23, 289.01, 288.43, 287.85, 287.27, 287.05, 286.84, 286.62, 286.34, 286.06, 285.78,
2 m agl Temp-D (K): -4.09, -4.41, -4.86, -5.36, -5.73, -5.96, -6.36, -6.16, -4.89, -3.72, -2.99, -2.32, -1.60, -1.10, -0.78, -0.99, -1.18, -2.06, -3.36, -4.23, -4.94, -5.57, -5.83, -6.11, -6.92, -7.37, -7.62, -7.74, -7.67, -7.75, -7.74, -7.22, -4.98, -2.51, -0.76, 0.23, 0.71, 1.19, 1.24, 1.76, 1.53, 0.95, 0.19, 0.17, 0.14, 0.12, 0.13, 0.14, 0.14, 0.16, 0.18, 0.20, 0.71, 1.21, 1.71, 2.73, 3.75, 4.77, 5.60, 6.43, 7.26, 7.91, 8.56, 9.21, 8.94, 8.67, 8.40, 7.96, 7.51, 7.07, 6.51, 5.96, 5.41, 4.87, 4.33, 3.79, 2.62, 1.46, 0.29, -0.35, -0.99, -1.63, -1.15, -0.67, -0.19, -0.42, -0.65, -0.88, -1.38, -1.88, -2.38, -2.66, -2.94, -3.22, -3.46, -3.69, -3.93, -4.07, -4.22, -4.36, -4.45, -4.53, -4.62, -4.23, -3.85, -3.46, -2.86, -2.25, -1.65, -1.22, -0.80, -0.37, -0.88, -1.39, -1.90, -2.15, -2.39, -2.64, -2.88, -3.11, -3.34, -3.50, -3.67, -3.83, -3.67, -3.51, -3.35, -2.89, -2.42, -1.96, -1.17, -0.37, 0.42, 1.22, 2.02, 2.82, 2.23, 1.63, 1.03, 0.80, 0.57, 0.34, 0.04, -0.25, -0.54,
2 m agl Dpt (K): 279.93, 279.97, 279.84, 279.84, 279.86, 279.92, 279.81, 280.17, 280.44, 280.24, 279.96, 280.01, 280.18, 280.19, 280.06, 279.51, 279.16, 278.73, 278.60, 278.48, 278.28, 278.02, 278.45, 278.95, 278.42, 278.22, 278.45, 278.63, 278.90, 278.83, 278.83, 279.16, 279.91, 280.54, 281.34, 282.05, 282.45, 283.00, 283.46, 284.32, 284.71, 284.89, 285.10, 285.45, 285.78, 286.12, 286.01, 285.91, 285.80, 285.62, 285.44, 285.26, 286.00, 286.73, 287.46, 288.58, 289.70, 290.82, 291.36, 291.90, 292.43, 292.36, 292.25, 292.11, 291.65, 291.18, 290.72, 290.58, 290.43, 290.27, 290.19, 290.10, 290.00, 289.28, 288.56, 287.84, 285.82, 283.78, 281.69, 281.59, 281.47, 281.33, 282.44, 283.53, 284.61, 284.41, 284.22, 284.03, 283.56, 283.09, 282.62, 282.14, 281.67, 281.19, 281.13, 281.08, 281.03, 281.07, 281.11, 281.15, 281.23, 281.31, 281.39, 281.43, 281.45, 281.47, 281.14, 280.74, 280.28, 280.28, 280.26, 280.23, 280.30, 280.36, 280.39, 280.62, 280.84, 281.03, 280.88, 280.72, 280.57, 280.84, 281.10, 281.34, 281.71, 282.08, 282.44, 282.87, 283.31, 283.74, 284.50, 285.26, 286.01, 286.48, 286.94, 287.38, 286.83, 286.27, 285.72, 285.31, 284.90, 284.49, 284.02, 283.56, 283.10,
2 m agl WBT (K): 281.58, 281.48, 281.17, 280.96, 280.76, 280.76, 280.45, 280.76, 281.48, 281.94, 282.09, 282.40, 282.81, 283.01, 283.12, 282.71, 282.50, 281.89, 281.27, 280.76, 280.35, 279.94, 280.04, 280.14, 279.48, 279.12, 279.12, 279.12, 279.32, 279.22, 279.27, 279.73, 281.17, 282.60, 283.73, 284.45, 284.86, 285.32, 285.58, 286.19, 286.35, 286.19, 286.04, 286.25, 286.40, 286.60, 286.50, 286.50, 286.40, 286.30, 286.19, 286.09, 286.71, 287.32, 287.94, 289.01, 290.09, 291.17, 291.78, 292.40, 292.96, 293.12, 293.22, 293.32, 292.91, 292.50, 292.14, 291.88, 291.68, 291.42, 291.17, 290.91, 290.70, 290.04, 289.42, 288.76, 287.17, 285.58, 284.04, 283.73, 283.43, 283.07, 283.83, 284.65, 285.48, 285.27, 285.06, 284.86, 284.45, 283.99, 283.53, 283.22, 282.81, 282.50, 282.40, 282.25, 282.09, 282.09, 282.04, 281.99, 281.99, 281.99, 282.04, 282.19, 282.40, 282.60, 282.71, 282.71, 282.76, 282.91, 283.12, 283.22, 283.01, 282.81, 282.60, 282.60, 282.60, 282.60, 282.40, 282.19, 281.99, 281.99, 282.09, 282.09, 282.35, 282.60, 282.81, 283.22, 283.63, 284.04, 284.81, 285.53, 286.30, 286.86, 287.42, 287.99, 287.42, 286.86, 286.30, 285.99, 285.68, 285.37, 284.96, 284.60, 284.25,
Convect. Temp (K): 303.57, 303.13, 303.20, 304.07, 303.98, 303.92, 304.01, 284.02, 283.59, 283.42, 283.80, 285.06, 285.45, 285.68, 286.01, 299.96, 299.77, 300.60, 301.14, 301.41, 301.44, 301.63, 301.08, 300.05, 297.39, 299.94, 300.80, 301.56, 301.54, 301.51, 301.55, 301.72, 302.09, 302.26, 302.30, 302.74, 302.65, 300.64, 299.55, 300.21, 301.11, 302.01, 302.01, 302.06, 302.12, 302.16, 302.32, 302.49, 302.63, 300.58, 300.13, 299.93, 300.10, 299.93, 299.32, 298.63, 297.88, 297.02, 296.63, 296.30, 296.06, 296.56, 296.92, 297.16, 297.80, 299.29, 302.04, 302.33, 302.37, 302.27, 300.75, 298.96, 297.27, 295.20, 293.51, 292.06, 290.58, 289.07, 287.64, 287.07, 286.42, 285.40, 285.87, 286.42, 286.81, 286.53, 286.27, 285.99, 285.79, 285.59, 285.40, 285.15, 284.89, 284.62, 284.51, 284.40, 284.27, 284.27, 284.27, 284.26, 284.12, 284.00, 283.89, 283.84, 283.66, 283.64, 284.14, 320.35, 320.32, 319.86, 319.53, 319.29, 319.07, 318.87, 318.69, 317.98, 317.16, 316.15, 315.87, 315.60, 315.35, 311.88, 309.30, 284.72, 285.32, 284.17, 284.02, 284.41, 284.82, 285.21, 285.91, 286.59, 287.26, 288.94, 289.49, 289.80, 289.48, 289.14, 288.80, 288.46, 288.12, 287.76, 287.43, 287.09, 286.73,
Heat Index (F): 50.62, 50.01, 49.18, 48.24, 47.60, 47.17, 46.47, 46.88, 49.17, 51.27, 52.57, 53.75, 55.01, 55.85, 56.39, 55.97, 55.60, 54.03, 51.69, 50.11, 48.84, 47.71, 47.22, 46.72, 45.26, 44.45, 43.98, 43.75, 43.88, 43.77, 43.80, 44.75, 48.80, 53.25, 56.37, 58.10, 58.89, 59.68, 59.67, 60.55, 60.11, 59.07, 57.72, 57.67, 57.61, 57.56, 57.54, 57.51, 57.49, 57.46, 57.44, 57.42, 58.30, 59.18, 60.06, 61.91, 63.77, 65.63, 67.04, 68.44, 69.84, 70.95, 72.05, 73.69, 73.37, 73.12, 72.95, 72.05, 69.87, 69.04, 68.03, 67.01, 66.00, 65.03, 64.06, 63.10, 61.05, 59.00, 56.95, 55.85, 54.76, 53.66, 54.53, 55.40, 56.26, 55.93, 55.59, 55.25, 54.47, 53.69, 52.90, 52.47, 52.03, 51.59, 51.20, 50.81, 50.41, 50.17, 49.93, 49.69, 49.57, 49.46, 49.35, 50.06, 50.78, 51.49, 52.51, 53.52, 54.53, 55.23, 55.93, 56.62, 55.70, 54.77, 53.85, 53.33, 52.81, 52.29, 51.79, 51.29, 50.79, 50.47, 50.14, 49.81, 49.93, 50.05, 50.16, 50.92, 51.67, 52.43, 53.73, 55.03, 56.33, 57.73, 59.13, 60.53, 59.48, 58.44, 57.39, 57.01, 56.62, 56.23, 55.73, 55.22, 54.71,
Wind Chill (F): 50.62, 50.01, 49.18, 48.24, 47.60, 47.17, 46.47, 46.88, 49.17, 51.27, 52.57, 53.75, 55.01, 55.85, 56.39, 55.97, 55.60, 54.03, 51.69, 50.11, 48.84, 47.71, 47.22, 46.72, 45.26, 44.45, 43.98, 43.75, 43.88, 43.77, 43.80, 44.75, 48.80, 53.25, 56.37, 58.10, 58.89, 59.68, 59.67, 60.55, 60.11, 59.07, 57.72, 57.67, 57.61, 57.56, 57.54, 57.51, 57.49, 57.46, 57.44, 57.42, 58.30, 59.18, 60.06, 61.91, 63.77, 65.63, 67.04, 68.44, 69.84, 70.95, 72.05, 73.16, 72.62, 72.07, 71.53, 70.70, 69.87, 69.04, 68.03, 67.01, 66.00, 65.03, 64.06, 63.10, 61.05, 59.00, 56.95, 55.85, 54.76, 53.66, 54.53, 55.40, 56.26, 55.93, 55.59, 55.25, 54.47, 53.69, 52.90, 52.47, 52.03, 51.59, 51.20, 50.81, 50.41, 50.17, 49.93, 49.69, 49.57, 49.46, 49.35, 50.06, 50.78, 51.49, 52.51, 53.52, 54.53, 55.23, 55.93, 56.62, 55.70, 54.77, 53.85, 53.33, 52.81, 52.29, 51.79, 51.29, 50.79, 50.47, 50.14, 49.81, 49.93, 50.05, 50.16, 50.92, 51.67, 52.43, 53.73, 55.03, 56.33, 57.73, 59.13, 60.53, 59.48, 58.44, 57.39, 57.01, 56.62, 56.23, 55.73, 55.22, 54.71,
FITS (F): 60.81, 60.34, 59.57, 58.79, 58.27, 57.96, 57.30, 57.87, 59.94, 61.55, 62.45, 63.46, 64.61, 65.31, 65.68, 64.98, 64.45, 62.87, 60.85, 59.47, 58.29, 57.19, 57.06, 56.96, 55.41, 54.61, 54.37, 54.30, 54.58, 54.44, 54.46, 55.46, 59.30, 63.38, 66.48, 68.36, 69.27, 70.28, 70.57, 71.85, 71.73, 70.98, 70.00, 70.18, 70.34, 70.52, 70.43, 70.34, 70.25, 70.12, 69.98, 69.85, 71.05, 72.25, 73.44, 75.70, 77.95, 80.21, 81.71, 83.22, 84.72, 85.59, 86.44, 87.26, 86.52, 85.77, 85.02, 84.25, 83.47, 82.68, 81.79, 80.89, 79.98, 78.72, 77.46, 76.20, 73.21, 70.21, 67.18, 66.21, 65.23, 64.23, 65.66, 67.07, 68.48, 68.07, 67.67, 67.27, 66.32, 65.37, 64.43, 63.76, 63.10, 62.43, 62.06, 61.71, 61.35, 61.17, 61.00, 60.82, 60.78, 60.74, 60.69, 61.31, 61.92, 62.52, 63.15, 63.74, 64.28, 64.86, 65.42, 65.98, 65.26, 64.53, 63.78, 63.50, 63.21, 62.91, 62.40, 61.88, 61.37, 61.27, 61.17, 61.05, 61.38, 61.71, 62.04, 62.94, 63.85, 64.75, 66.31, 67.87, 69.43, 70.89, 72.34, 73.78, 72.56, 71.34, 70.12, 69.54, 68.96, 68.38, 67.66, 66.94, 66.23,
2 m agl RH (%): 78.52, 80.53, 82.33, 85.25, 87.44, 89.24, 90.92, 91.78, 85.81, 78.26, 73.18, 70.31, 67.95, 65.96, 64.10, 62.67, 62.01, 63.72, 68.82, 72.37, 74.81, 76.64, 80.45, 84.87, 86.47, 87.96, 90.96, 92.91, 94.26, 94.21, 94.08, 92.82, 83.92, 74.29, 70.00, 69.04, 68.95, 69.56, 71.75, 73.64, 76.72, 80.61, 85.75, 87.91, 90.06, 92.22, 91.68, 91.14, 90.59, 89.61, 88.62, 87.63, 89.11, 90.59, 92.06, 92.62, 93.18, 93.74, 92.40, 91.05, 89.70, 85.99, 82.29, 78.58, 77.76, 76.93, 76.10, 77.56, 79.02, 80.49, 82.95, 85.42, 87.88, 86.82, 85.76, 84.70, 79.88, 75.06, 70.24, 72.57, 74.90, 77.23, 80.66, 84.08, 87.50, 87.45, 87.40, 87.36, 87.11, 86.86, 86.61, 85.24, 83.87, 82.51, 83.42, 84.34, 85.25, 86.28, 87.30, 88.32, 89.19, 90.05, 90.92, 88.76, 86.60, 84.44, 79.51, 74.57, 69.63, 67.86, 66.08, 64.31, 66.85, 69.39, 71.92, 74.47, 77.02, 79.57, 80.21, 80.85, 81.48, 84.02, 86.55, 89.08, 90.95, 92.82, 94.69, 94.81, 94.93, 95.05, 95.29, 95.52, 95.76, 93.86, 91.96, 90.06, 90.16, 90.27, 90.38, 89.22, 88.06, 86.90, 85.84, 84.78, 83.72,
10 m agl Dir: 298.95, 302.50, 308.79, 299.63, 292.80, 286.98, 268.80, 267.70, 268.67, 280.31, 288.14, 292.69, 297.80, 302.88, 311.08, 319.73, 325.41, 345.53, 14.91, 55.37, 85.08, 111.53, 154.23, 180.59, 139.45, 94.70, 99.73, 76.02, 82.79, 98.78, 110.05, 119.49, 125.92, 125.22, 131.40, 135.67, 143.81, 156.04, 149.98, 151.57, 138.75, 125.78, 126.24, 115.52, 106.67, 99.56, 108.58, 117.48, 125.86, 141.16, 155.55, 167.60, 169.99, 172.00, 173.71, 173.77, 173.84, 173.91, 173.34, 172.80, 172.27, 176.46, 180.11, 183.29, 181.13, 179.30, 177.74, 178.17, 178.56, 178.91, 178.99, 179.09, 179.20, 192.90, 211.74, 233.35, 238.35, 243.32, 248.17, 248.51, 248.85, 249.21, 256.61, 264.64, 273.04, 282.95, 290.69, 296.70, 304.82, 313.55, 322.54, 324.25, 326.27, 328.69, 326.94, 325.09, 323.14, 325.79, 328.76, 332.07, 332.09, 332.12, 332.14, 343.91, 357.24, 10.88, 25.49, 47.55, 73.75, 86.61, 98.64, 109.03, 114.34, 119.20, 123.59, 120.95, 119.13, 117.81, 119.46, 121.32, 123.43, 124.21, 125.26, 126.74, 130.31, 132.09, 133.15, 135.12, 137.25, 139.57, 143.71, 148.38, 153.60, 176.75, 205.49, 228.70, 232.07, 234.95, 237.43, 249.01, 259.06, 267.38, 268.20, 269.05, 269.92,
10 m agl Spd (kt): 4.00, 4.00, 4.00, 4.00, 4.00, 4.00, 4.00, 4.00, 5.00, 7.00, 7.00, 7.00, 7.00, 7.00, 7.00, 6.00, 5.00, 4.00, 4.00, 3.00, 2.00, 3.00, 3.00, 4.00, 1.00, 3.00, 3.00, 3.00, 3.00, 4.00, 4.00, 4.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 7.00, 5.00, 5.00, 5.00, 5.00, 5.00, 6.00, 6.00, 6.00, 6.00, 6.00, 7.00, 8.00, 8.00, 9.00, 10.00, 10.00, 10.00, 10.00, 10.00, 10.00, 10.00, 11.00, 12.00, 13.00, 14.00, 15.00, 16.00, 17.00, 18.00, 19.00, 18.00, 17.00, 15.00, 13.00, 11.00, 11.00, 11.00, 11.00, 11.00, 11.00, 11.00, 11.00, 10.00, 10.00, 10.00, 11.00, 13.00, 14.00, 14.00, 13.00, 13.00, 12.00, 11.00, 10.00, 10.00, 10.00, 9.00, 9.00, 8.00, 8.00, 7.00, 7.00, 6.00, 6.00, 6.00, 6.00, 5.00, 4.00, 4.00, 4.00, 4.00, 5.00, 5.00, 5.00, 5.00, 6.00, 8.00, 9.00, 8.00, 8.00, 7.00, 6.00, 6.00, 5.00, 7.00, 9.00, 12.00, 11.00, 11.00, 10.00, 10.00, 9.00, 9.00, 7.00, 7.00, 9.00, 9.00, 10.00, 11.00, 11.00, 12.00, 14.00, 14.00, 13.00, 13.00,
###....line 160UA SECTION###
1000mb GPH (m): 166.88, 165.16, 162.34, 159.50, 161.16, 160.70, 161.16, 164.98, 165.70, 164.82, 163.34, 162.34, 158.34, 153.52, 150.18, 147.18, 145.72, 145.72, 145.84, 144.54, 146.00, 146.76, 144.64, 144.12, 144.90, 144.54, 143.18, 141.84, 142.66, 143.94, 144.94, 146.66, 148.78, 148.96, 145.84, 141.84, 136.12, 128.66, 120.44, 116.66, 114.50, 113.42, 116.02, 114.77, 113.53, 112.28, 109.00, 105.72, 102.44, 97.50, 92.56, 87.62, 85.30, 82.99, 80.68, 82.66, 84.64, 86.62, 78.45, 70.28, 62.12, 55.95, 49.78, 43.62, 38.65, 33.68, 28.71, 26.56, 24.41, 22.25, 20.78, 19.30, 17.82, 18.34, 18.86, 19.38, 24.23, 29.07, 33.92, 39.10, 44.28, 49.46, 49.72, 49.99, 50.26, 56.58, 62.90, 69.22, 79.36, 89.50, 99.64, 105.31, 110.98, 116.64, 119.50, 122.35, 125.20, 126.42, 127.64, 128.86, 132.69, 136.51, 140.34, 142.01, 143.67, 145.34, 139.01, 132.67, 126.34, 120.35, 114.37, 108.38, 107.47, 106.57, 105.66, 99.96, 94.27, 88.58, 81.61, 74.65, 67.68, 64.92, 62.16, 59.39, 44.82, 30.25, 15.68, 8.73, 1.77, -5.18, -16.35, -27.52, -38.68, -42.91, -47.13, -51.36, -48.50, -45.64, -42.78, -39.97, -37.16, -34.34, -32.48, -30.62, -28.76,
1000mb GPH DVal(m): 56.00, 54.28, 51.46, 48.62, 50.28, 49.82, 50.28, 54.09, 54.82, 53.93, 52.46, 51.46, 47.46, 42.64, 39.30, 36.30, 34.84, 34.84, 34.96, 33.66, 35.12, 35.87, 33.75, 33.23, 34.02, 33.66, 32.30, 30.96, 31.77, 33.05, 34.05, 35.77, 37.89, 38.07, 34.96, 30.96, 25.23, 17.77, 9.55, 5.77, 3.61, 2.53, 5.14, 3.89, 2.64, 1.40, -1.88, -5.17, -8.45, -13.39, -18.33, -23.27, -25.58, -27.89, -30.21, -28.23, -26.25, -24.27, -32.43, -40.60, -48.77, -54.94, -61.10, -67.27, -72.24, -77.20, -82.17, -84.33, -86.48, -88.63, -90.11, -91.59, -93.07, -92.55, -92.02, -91.50, -86.66, -81.81, -76.97, -71.79, -66.61, -61.43, -61.16, -60.89, -60.63, -54.30, -47.98, -41.66, -31.52, -21.38, -11.24, -5.58, 0.09, 5.76, 8.61, 11.46, 14.32, 15.54, 16.76, 17.98, 21.80, 25.63, 29.46, 31.12, 32.79, 34.46, 28.12, 21.79, 15.46, 9.47, 3.48, -2.50, -3.41, -4.32, -5.23, -10.92, -16.61, -22.31, -29.27, -36.24, -43.20, -45.97, -48.73, -51.49, -66.06, -80.63, -95.20, -102.16, -109.11, -116.07, -127.23, -138.40, -149.57, -153.79, -158.02, -162.25, -159.39, -156.53, -153.67, -150.85, -148.04, -145.23, -143.37, -141.51, -139.65,
1000mb Temp (K): 283.88, 283.48, 283.09, 282.57, 282.16, 281.83, 281.51, 281.33, 282.10, 282.98, 283.56, 284.11, 284.78, 285.36, 285.80, 285.93, 285.93, 285.70, 284.88, 284.21, 283.76, 283.39, 282.76, 282.08, 281.82, 281.54, 281.03, 280.67, 280.38, 280.28, 280.23, 280.44, 282.13, 284.26, 285.90, 286.92, 287.64, 288.15, 288.42, 288.76, 288.83, 288.58, 288.07, 287.99, 287.90, 287.81, 287.87, 287.93, 287.98, 288.04, 288.09, 288.15, 288.63, 289.10, 289.58, 290.41, 291.24, 292.07, 292.93, 293.79, 294.66, 295.37, 296.08, 296.79, 296.65, 296.51, 296.37, 295.86, 295.34, 294.83, 294.26, 293.69, 293.13, 292.67, 292.20, 291.74, 290.46, 289.18, 287.91, 287.13, 286.35, 285.57, 286.00, 286.42, 286.84, 286.70, 286.56, 286.43, 285.97, 285.52, 285.07, 284.78, 284.49, 284.20, 283.95, 283.70, 283.45, 283.31, 283.17, 283.03, 282.94, 282.84, 282.75, 283.01, 283.27, 283.53, 284.01, 284.50, 284.98, 285.48, 285.98, 286.48, 286.25, 286.02, 285.79, 285.50, 285.22, 284.93, 284.73, 284.53, 284.34, 284.12, 283.91, 283.70, 283.85, 284.01, 284.16, 284.62, 285.07, 285.52, 286.27, 287.02, 287.77, 288.59, 289.41, 290.24, 289.73, 289.22, 288.72, 288.49, 288.27, 288.05, 287.72, 287.40, 287.07,
1000mb Temp DVal(K): -3.56, -3.96, -4.35, -4.87, -5.28, -5.61, -5.93, -6.11, -5.34, -4.46, -3.88, -3.33, -2.66, -2.08, -1.64, -1.51, -1.51, -1.74, -2.56, -3.23, -3.68, -4.05, -4.68, -5.36, -5.62, -5.90, -6.41, -6.77, -7.06, -7.15, -7.21, -7.00, -5.31, -3.17, -1.53, -0.51, 0.20, 0.71, 0.98, 1.32, 1.39, 1.14, 0.63, 0.55, 0.46, 0.37, 0.43, 0.49, 0.55, 0.60, 0.65, 0.71, 1.19, 1.67, 2.14, 2.97, 3.80, 4.63, 5.49, 6.35, 7.22, 7.93, 8.64, 9.36, 9.21, 9.07, 8.93, 8.42, 7.90, 7.39, 6.82, 6.26, 5.69, 5.23, 4.76, 4.30, 3.02, 1.74, 0.47, -0.31, -1.09, -1.87, -1.44, -1.02, -0.60, -0.74, -0.88, -1.01, -1.47, -1.92, -2.37, -2.66, -2.95, -3.24, -3.49, -3.74, -3.99, -4.13, -4.27, -4.41, -4.50, -4.59, -4.68, -4.43, -4.17, -3.91, -3.43, -2.94, -2.45, -1.96, -1.46, -0.96, -1.19, -1.42, -1.65, -1.94, -2.22, -2.51, -2.71, -2.91, -3.10, -3.32, -3.53, -3.74, -3.59, -3.43, -3.28, -2.82, -2.37, -1.92, -1.17, -0.42, 0.33, 1.15, 1.97, 2.80, 2.29, 1.78, 1.28, 1.06, 0.83, 0.61, 0.28, -0.04, -0.37,

• Teepeemm: Is all your Python in one file? Could you format it as such here? Could you give a few more lines of your text file, so that we could copy and paste it into our own copy of the text file to run your script?
• Jason Leaver: I updated the code with the full context, you should be able to save that text file in a folder called data/2021102206Z.txt and call run_test()

Answer 1

• pass is for when Python requires you to have something, but there's nothing else to have. So it works in the except clauses, but otherwise isn't necessary.
• But catching an exception without doing anything else is usually a bad idea. You want to know that something happened, especially if you're dealing with sloppy data. The only occurrence I ran into was an IndexError on the ### line. I added a conditional to avoid that, and removed the except clauses.
• Classes automatically inherit from object; you don't need to do so explicitly.
• Don't use list(map) instead of a for loop in Tarp.__init__. Generally, comprehensions are better than map.
• Having _pythonic_keys sometimes return a str and sometimes False is a bit odd. Since you're testing its truthiness, lets return None instead of False.
• But actually, it turns out that _pythonic_key doesn't need its if check. If the string is only whitespace, then (formerly) v ended up empty which caused an IndexError and nothing happened. Now, key ends up empty for a quick return.
• I find it much easier to read a simple regular expression than to remember what it's doing from the variable name. One could argue that compiling them gives a bit of efficiency, but (1) that's way too premature of an optimization and (2) Python caches its regexs, so it doesn't end up being an optimization anyway.
• Object is a terrible name for a class. Maybe KeysList? But for that matter, the only point of the props and models property is for the key/value object that it's keeping. Python already does that with dicts, so lets use dicts instead. (If you really needed a key/value object, a Named Tuple could do the job.)
• Now we realize that the zip functions are really just returning the dicts, so lets do that instead. Also, you never used zip_props. And asMatrix is just using the dict values. This eliminates the need for getmodels and getprops.
• You ended up with forecast_hours in props with a value that was everything else. I assume that was a bug, and moved it to models.
• Single letter variables are ok for a short time and if their meaning is clear. k and v (kind of) pass this test, but s and f don't.
• Overwriting v makes it a bit harder to debug. But val doesn't gain you much, so I repurposed val to be the new v (and got rid of the old val).
• _pythonic_vals does not behave like _pythonic_keys. It seems that the main purpose is to remove the final entry. But it shouldn't unilaterally do that without looking at that final entry.
• _series takes a line of text, constructs a length 1 series, and then splits it. Don't go through pandas just to split. I had a hard time understanding this method in general, as it had several of these items that I changed.
• In a similar spirit of avoiding unnecessary methods, I would avoid using regex when you can do the same thing with string methods (eg, _only_wspace and _units).
• Your comment says that _series splits on : or , or whitespace, but it's really : or , and strips whitespace after that. I would consider that statement too obvious to comment, and you definitely don't want a comment to be in conflict with the code.
• _pythonic_key is only called from within Tarp, so I moved it there. I also made it a staticmethod.
• I added type hints, so that you can run python -m mypy thisfile.py.
• I added a __main__ guard, so that this could end up in its own package. I also added a few more print statements to run_test().
• You switch between " and '. If there's a reason for that, that would be fine. If not, you should be more consistent.

import numpy as np # type: ignore
import pandas as pd # type: ignore
import re
from datetime import datetime
from typing import Dict,List,TextIO,Union
class Tarp:
 def __init__(self, filein: TextIO):
 self.props: Dict[str,Union[str,datetime]] = {}
 self.models: Dict[str,np.array] = {}
 for line in filein:
 self._series(line)
 @staticmethod
 def _pythonic_key(s:str) -> str:
 return re.sub(r'\s+','_',s.split('(',1)[0].rstrip()).lower()
 def _series(self, line: str) -> None:
 rowentries: List[str] = re.split(r':\s*|,\s*',line)
 key: str = self._pythonic_key(rowentries.pop(0))
 if rowentries:
 if not re.search(r'\S',rowentries[-1]):
 rowentries.pop()
 if not key or not rowentries:
 return
 if len(rowentries)>1:
 self.models[key] = np.array(rowentries, dtype=float)
 else:
 val: str = rowentries[0].rstrip()
 if key == 'basetime':
 self.props[key] = datetime.strptime(val, '%Y%m%d%HZ')
 elif key == 'forecast_hours':
 self.models[key] = np.array( [ t.rstrip('hr') for t in val.split() ] ,
 dtype=int)
 else:
 self.props[key] = val
 def asMatrix(self) -> np.array:
 return np.array( self.models.values() )
 def asDataFrame(self) -> pd.DataFrame:
 return pd.DataFrame(self.models)
def run_test():
 with open('data/2021102206Z.txt', 'r') as filein:
 tp = Tarp(filein)
 print('props:',tp.props)
 df = tp.asDataFrame()
 print(df.shape)
 print(df.dtypes)
 print(type(df))
 print(df)
 m = tp.asMatrix()
 print(type(m))
if __name__ == '__main__':
 run_test()