Foldify - A Python Folder Tree Manager Tool

Question

The goal was to create a tool to help manage folders structures, and allow me to create templates of this folders and store them as a JSON template. The following functionalities have been implemented:

1. Make a copy of a directory tree (folders only).
2. Create a JSON template of the directory tree.
3. Create a directory tree from a JSON template.

There are 3 modules:

1. path_tree.py: path object class to help navigate directory and turn it into a tree object.
2. core.py: main functionalities, handles exceptions
3. foldify.py: CLI interface for executing the tasks

I would appreciate any feedback on improving the code or its organization.

Repo:

[email protected]:gtalarico/foldify.git

https://github.com/gtalarico/foldify

path_tree.py: a PathObject (PathO) helper class to help deal with folder hierarchy relationships (children, iterate up and down, get root, etc)

import os
import json
from collections import OrderedDict
class PathO(object):
 def __init__(self, name, static_fullpath=None, children=[], parent=None):
 self.name = name
 self.children = children
 self.parent = parent
 self.static_fullpath = static_fullpath
 self.path_type = None
 @property
 def exists(self):
 return os.path.exists(self.static_fullpath)
 @property
 def isdir(self):
 if self.exists:
 return os.path.isdir(self.static_fullpath)
 @property
 def isfile(self):
 if self.exists:
 return not self.isdir
 # @property
 def get_path_type(self):
 if self.isdir:
 self.path_type = 'folder'
 return self.path_type
 elif self.isfile:
 self.path_type = 'file'
 return self.path_type
 @property
 def ancestors(self):
 return [a for a in self.iter_up()]
 @property
 def ancestors_fullpath(self):
 """Similar to fullpath, but it's build from path ancestors"""
 return os.path.join(*[x.name for x in reversed(self.ancestors)])
 @property
 def root(self):
 return self.ancestors[-1]
 def iter_up(self):
 ''' Iterates upwards: yields self first, and ends with root
 Does not iterate over cousings or ancestors not in a direct inheritance
 line towards root
 '''
 yield self
 if self.parent is None:
 pass
 else:
 for parent in self.parent.iter_up():
 yield parent
 def iter_down(self):
 ''' Iterates downwards
 yields self first, then iterates over
 its children's children recursevely
 ending with last lowest child
 '''
 yield self
 for child in self.children:
 # yield child
 for c in child.iter_down():
 yield c
 def get_json_dict(self, detailed=False):
 d = OrderedDict()
 d['name'] = self.name
 d['type'] = self.path_type
 d['children'] = [x.get_json_dict() for x in self.children]
 if detailed:
 d['parent'] = getattr(self.parent, 'name', None)
 return d
 def get_json_string(self):
 return json.dumps(self.get_json_dict(), encoding='utf-8',
 ensure_ascii=False, sort_keys=False, indent=2,
 separators=(',', ': '))
 def __repr__(self):
 return '<PATH:{0}|PARENT:{1}|CHILDS:{2}>'.format(
 self.name,
 getattr(self.parent,'name', None),
 len(self))
 def __len__(self):
 '''Returns number of children, files or folders'''
 if self.children:
 return len(self.children)
 else:
 return 0
def tree_from_folder(source_folder):
 ''' creates patho tree of patho objects from a local folder name'''
 patho = PathO(os.path.basename(source_folder), static_fullpath=source_folder)
 patho.get_path_type()
 try:
 patho.children = [tree_from_folder(os.path.join(source_folder,x)) for x in os.listdir(source_folder)]
 except OSError as errmsg:
 pass # if is file, listdir will fail
 else:
 for child in patho.children:
 child.parent = patho
 return patho
def tree_from_json_dict(json_dict):
 ''' creates a PathO tree from a json_dict 
 (matching the jsson created by a PathO tree)'''
 patho = PathO(json_dict['name'])
 patho.path_type = json_dict['type']
 # import pdb; pdb.set_trace()
 try:
 patho.children = [tree_from_json_dict(x) for x in json_dict['children']]
 except KeyError:
 pass
 else:
 for child in patho.children:
 child.parent = patho
 return patho

core.py Functions to help with basic tasks, and manage exceptions

import json
import sys
import os
import shutil
import copy
from path_tree import tree_from_folder, tree_from_json_dict
# TO DO:
# Add Tests 
def load_file(source_file):
 ''' loads a json, returns a python list/dictionary object'''
 try:
 with open(source_file, 'r') as f:
 try:
 return json.load(f)
 except ValueError as e:
 print("Could Not Parse Json: {}".format(e))
 except IOError as errmsg:
 print errmsg
# tree = load_file('test.json')
def dump_json(filename, json_dict):
 ''' creates a .json file from a python object '''
 filename = '{0}.json'.format(filename)
 try:
 with open(filename, 'wx') as outfile:
 json.dump(json_dict, outfile, indent=2)
 except IOError as errmsg:
 print errmsg
 else:
 return True
# dump_json('test.json', tree.get_json_dict())
def mkdirs_from_json_dict(new_foldername, json_dict):
 if os.path.exists(new_foldername):
 print 'Cannot Copy. Folder already exists: ', new_foldername
 return
 new_tree = tree_from_json_dict(json_dict)
 new_tree.root.name = new_foldername # set rootname of new folder
 failed = False
 for patho in new_tree.iter_down():
 if patho.path_type == 'folder':
 try:
 os.makedirs(patho.ancestors_fullpath)
 except OSError as errmsg:
 import pdb; pdb.set_trace()
 print errmsg
 failed = True
 break
 else:
 try:
 with open(patho.ancestors_fullpath, 'w') as f:
 pass
 except OSError as errmsg:
 print errmsg
 failed = True
 break
 if not failed and os.path.exists(new_tree.root.ancestors_fullpath):
 return True
 else:
 print 'Make Dirs Operation Failed Deleting tree: ', new_foldername
 try:
 shutil.rmtree(new_tree.root.fullpath)
 except:
 print 'Attempted but failed to delete folder: ', new_foldername
def mkjson_from_folder(source_folder):
 """Returns Json_dict from folder, None if Folder not found or error"""
 if os.path.isdir(source_folder):
 tree = tree_from_folder(source_folder)
 return tree.get_json_dict()
 else:
 print 'Failed to make json. Folder not found: [{}]'.format(source_folder)

foldify.py Run this to see tools in action: a command line interface for the user.

import sys
from collections import OrderedDict
from core import mkjson_from_folder, mkdirs_from_json_dict
from core import load_file, dump_json
def menu_copy_folder_tree():
 """Copy Folder Tree."""
 source_folder = raw_input('Name of Source Folder: \n>>>')
 dest_folder = raw_input('Name of Destination Folder (Blank for X-copy): \n>>>')
 if dest_folder == '':
 dest_folder = '{}_copy'.format(source_folder)
 json_dict = mkjson_from_folder(source_folder)
 if json_dict and mkdirs_from_json_dict(dest_folder, json_dict):
 print 'Folder Structure of [{0}] successfully copied to [{1}]'.format(
 source_folder, dest_folder)
def menu_json_from_folder():
 """Make Json from Folder."""
 source_folder = raw_input('Name of Source Folder: \n>>>')
 dest_file = raw_input('Name of JSON (.json will be added; blank for same name): \n>>>')
 if dest_file == '':
 dest_file = source_folder
 json_dict = mkjson_from_folder(source_folder)
 if json_dict and dump_json(dest_file, json_dict):
 print 'New Json template created for folder [{}]'.format(source_folder)
def menu_folder_from_json():
 """Make Folder from Json."""
 source_file = raw_input('Name of Source JSON: \n>>>')
 dest_folder = raw_input('Name of Destination Folder (Leave Blank to try use json root): \n>>>')
 json_dict = load_file(source_file)
 if dest_folder == '':
 dest_folder = json_dict['name']
 if json_dict and mkdirs_from_json_dict(dest_folder, json_dict):
 print 'New folder [{}] created from json [{}]'.format(dest_folder,
 source_file)
def menu_exit():
 """Exit the program."""
 sys.exit()
menu = (
 ('1', menu_copy_folder_tree),
 ('2', menu_json_from_folder),
 ('3', menu_folder_from_json),
 ('4', menu_exit)
 )
menu = OrderedDict(menu)
while True:
 print '='*30
 print 'Foldify'
 print '='*30
 for n, func in menu.items():
 print '{0} - {1}'.format(n, func.__doc__)
 selection = raw_input('Select an option:')
 try:
 menu[selection]()
 except KeyError:
 print 'Invalid Option'

Answer 1

You need to re-think your design. Make two halves of this, one is the API and should be programmed like an API, the other is the client code that uses this API. Do not ever mix the two. You have prints in the API part of the code that should really be in the client code part.

First off, you should formalize how you make your tree. I'd recommend that you convert the file system to the same way you would read the JSON. This is actually really simple. You get the name, and type of the node that you are at. You then get the nodes children if it has any.

And so I'd do something like:

# Globals
class FT: # FileType
 FOLDER = 'folder'
 FILE = 'file'
 OTHER = 'N/A'
class DN: # DataName
 NAME = 'name'
 TYPE = 'type'
 CHILDREN = 'children'
def _get_type(path):
 exists = os.path.exists
 isdir = os.path.isdir
 if not exists(path):
 return FT.OTHER
 if isdir(path):
 return FT.FOLDER
 return FT.FILE
def _read_path(path):
 obj = {
 DN.NAME: os.path.basename(path),
 DN.TYPE: _get_type(path)
 }
 if obj[DN.TYPE] == FT.FOLDER:
 join = os.path.join
 obj[DN.CHILDREN] = [
 _read_path(join(path, file_name))
 for file_name in os.listdir(path)
 ]
 return obj

This will give the same output as if we open the JSON file. Which can be simplified to:

def _read_json(path):
 with open(path, 'r') as f:
 return json.load(f)

After formalizing the input to the tree, you can now make the tree. The tree that you are making is most likely a B-Tree and so I'd call it that. I'm keeping things simple, as your code is overcomplicated to the point that I'd recommend re-writing it.

Anyway, the Node can create all it's children as it'll simplify the creation to just passing the object we get from the above functions. And then we want to be able to get a dict from it. And so the nodes can be:

class Node(object):
 def __init__(self, data):
 self.name = data.get(DN.NAME, None)
 self.parent = None
 self.type = data.get(DN.TYPE, FT.OTHER)
 self.children = [
 Node(child)
 for child in data.get(DN.CHILDREN, [])
 ]
 for child in self.children:
 child.parent = self
 def dict(self):
 return OrderedDict({
 DN.NAME: self.name,
 DN.TYPE: self.type,
 DN.CHILDREN: [c.dict() for c in self.children]
 })

After this I'd make a helper class to interact with the nodes. This class should also allow a way to save the nodes either as JSON or in the file system. Both can be simple.

class BTree(object):
 def __init__(self, path, json=False):
 fn = _read_json if json else _read_path
 data = fn(path)
 self.root = Node(data)
 def write_json(self, path):
 with open(path, 'w') as f:
 json.dump(self.root.dict(), f)
 def write_files(self, path):
 join = os.path.join
 path, name = os.path.split(path)
 self.root.name = name
 def build(path, node):
 path = join(path, node.name)
 if node.type == FT.FOLDER:
 os.makedirs(path)
 elif node.type == FT.FILE:
 with open(path, 'a') as f:
 pass
 for child in node.children:
 build(path, child)
 build(path, self.root)

This now makes usage much easier than before. BTree(src).write_json(dest). You can then add more to the BTree to allow changes to the data. And if you want to change the data, only do it through BTree and Node. Do not change it when you load or save the data, these should be pure 1:1 changes of the Tree to the desired format. (And so I'd remove the two lines that do this in write_files, but I kept them to be consistent with your current code.)

Finally you can then simplify their usage in you client code. menu_copy_folder_tree, without error handling, can be:

def menu_copy_folder_tree():
 """Copy Folder Tree."""
 src = raw_input('Name of Source Folder: \n>>>')
 dest = raw_input('Name of Destination Folder (Blank for X-copy): \n>>>')
 if dest == '':
 dest = '{}_copy'.format(src)
 BTree(src).write_files(dest)
 print 'Folder Structure of [{0}] successfully copied to [{1}]'.format(src, dest)

You really need to aim for a less is more approach and separate program from API.

• gtalarico: Thank you for your input Joe, I appreciate it. This is a practice project for me, so will take your input and try to Re write it. I started learning python as my 1st language recently. I'm comfortable with the syntax but I definitely have a long way to go on design patterns and code organization. Thanks again!
• Peilonrayz: @gtalarico No problem, as a beginner your grasp of the language is pretty good, as you don't seem like one! As I didn't know you were a beginner I skipped a few things I'd thought you've already covered, and so my answer may be lacking. Best of luck, :)
• gtalarico: No worries. My fault, I should have mentioned it. I read through your code and it made perfect sense. It was very helpful to see brand new solution form scratch from someone who actually knows what they are doing.