core_gather
"""
Copyright (C) 2020 New Entity Operations Inc.
ALL RIGHTS RESERVED
Abstract
--------
core_gather gives the operator methods for locating .entity/.ds relationships with dlist_drill()
Also, you're able to edit and add file types to search for, and parsing options if you know the
add on packages command syntax.
A File-type, Path, and Key are identified by system programs and looked up
"""
## Imports: Standard
from glob import (
glob,
)
## Imports: Community
from pandas import (
read_csv,
)
## import relevant settings
from core_middlelayer import (
asset_folder, blank, definition,
divisor,
ds_extention, entity_extension,
entity_dskey, entity_key,
openpackager_folder,
question1, question2, question3,
recursive_set_true,
replace_entity_title_back, replace_entity_title_front,
replace_ds_title_back, replace_ds_title_front,
replace_list_index
)
## Start with empty type lists - here, it's 'dlist_list' and 'entity_list'
dlist_list = []
entity_list = []
## default index-postion = 0
index_starter_count = 0
## assign paths
datascript_file = str(asset_folder)+str(openpackager_folder)+str(definition)+str(ds_extention)
datascript_file_exact = str(asset_folder)+str(openpackager_folder)+str(definition)
entity_file = str(asset_folder)+str(openpackager_folder)+str(definition)+str(entity_extension)
## craw definitions
dfile = glob(str(datascript_file), recursive=recursive_set_true)
efile = glob(str(entity_file), recursive=recursive_set_true)
## Basic core_gather operating classes
class core_gather_entity:
"""
core_gather_entity produces the list 'entity_list'
"""
entity_list = []
for file in efile:
input = open(file, 'r')
for line in input:
if entity_key in line:
line_needed = line
cleaned_line = line_needed.replace(replace_entity_title_front,blank)
formatted_line = cleaned_line.replace(replace_entity_title_back,blank)
field_result = formatted_line
entity_list_content = entity_list.append(field_result)
print(entity_list)
class core_ds_gather_by_entity:
"""
core_ds_gather_by_entity produces the list 'dlist_list'
"""
dlist_list = []
for file in efile:
input = open(file, 'r')
for line in input:
if entity_dskey in line:
line_needed = line
cleaned_line = line_needed.replace(replace_ds_title_front,blank)
formatted_line = cleaned_line.replace(replace_ds_title_back,blank)
field_result = formatted_line
dlist_list_content = dlist_list.append(field_result)
print(dlist_list)
def entity_construct():
"""
Runs when dlist_drill() runs
"""
# Perform base instance reads
global entities
entities = read_csv(func_a)
print("loaded entities")
# Load the standardized access Interface according to labels
# Gives a RangeIndex(start=0, stop=50, step=1)
global index
index = entities.index
# Load columns according to a Index(['id', 'value1', '...'], dtype='X'])
global columns
columns = entities.columns
# Load an array([[1, 'X', 'Y']])
# 'data' defaults to an array
global data
data = entities.values
# Gives a raw number of the frame block quantity
global total_block_size
total_block_size = entities.size
# Returns a (X, Y) mapping in raw counted format
global block_shape
block_shape = entities.shape
# Returns a full listed value representation of the new entity
# offers a linkable bound method
global block_all
block_all = entities.describe
# Gives a numerical analysis of the Entity
global sage
sage = entities.describe()
# Understnad the entity
#global entity_knowledge
#entity_knowledge = ###NONE PROVIDED###
# Map the int only columns: Snapshot to mark the a frames position
global entity_snap_ints
entity_snap_ints = entities.select_dtypes(include=['int']).head()
# Map the int only columns: Full
global entity_ints_full
entity_ints_full = entities.select_dtypes(include=['int'])
# How many Data objects are in the Frame's columns?
global entity_count
entity_count = entities.count()
# More detailed entity numbers/math logic
global entity_deep_stats
entity_deep_stats = entities.describe(percentiles=[.01, .1, .2, .3, .5, .95, .99])
# Check and replace null/blank values with a returned DataFrame view object
global entity_none_fill
entity_none_fill = entities.select_dtypes(['object']).fillna('')
# Conversion Area: Storing and Manipulating DataFrames
# Get a raw numerical opportunity to handel data[0, 1, 2, XN]
global index_list
index_list = entities.index.tolist()
# Get a raw list ['id', 'X', 'X1']
global columns_list
columns_list = entities.columns.tolist()
# This will return the same as the columns_list above,
# but you can adjust the index 0 as needed
global columns_as_cleaned_list
columns_as_cleaned_list = columns_list[0:]
global entity_info
def entity_info():
"""
This should be ran as .entity_info()
The function has no immediate need for a runtime and should be thought
of as being read only.
"""
print(entities.info())
global entity_show_all
def entity_show_all():
"""
This will take the stored dlist slug bucket, sort it, and return the
values including hidden files available and those without direct
relationships. All returned paths are full-paths
"""
# Establish counters
counter_show_all = 0
z = dfile.sort()
print("Your entities are here.")
for i in dfile:
counter_show_all = counter_show_all
string_counter = str(counter_show_all)
print(string_counter+" :"+i)
# Increase the counter object
counter_show_all+=1
print("Try dlist_drill() to load in an entity.")
global entity_column_memory
def entity_column_memory():
"""
This should be ran as .entity_memory()
This will give you a mb usage of the currently loaded frame
"""
print(entities.columns.memory_usage(deep=True))
global fine_tune
def fine_tune(col, value):
"""
Get finer entity details by col, value calls
core_gather.fine_tune('id', 2)
...
"""
print(entities[entities[col] == value])
# Allow for dynamic seeking
global seeker_selector
def seeker_selector():
"""
Navigate a structure according to required selector steps
"""
for i, item in enumerate(columns_list, start=index_starter_count):
print(i,str(divisor),item)
seek = int(input(question2))
print(seek)
seek_result = columns[seek]
defined_seek = entities[seek_result].unique()
print(defined_seek)
print(seek)
global seek_context
seek_context = input(question3)
print('You Selected to Display',seek_context)
global fine_tune_seek
if seek_context.isdigit():
fine_tune_seek = entities[entities[seek_result] == int(seek_context)]
print(fine_tune_seek)
else:
fine_tune_seek = entities[entities[seek_result] == str(seek_context)]
print(fine_tune_seek)
## Establish your drills
def dlist_drill(object=None):
"""
dlist_drill(object) creates a runtime for finding your .entity files
.entity files are then used for deeper search index-retrieval functions
if object == None: you'll do a full index-tree search.
if you pass an 'object', it should be in the form of DS5555555.entity
"""
if object == None:
"""
Standard full index-tree search
"""
for i, item in enumerate(core_gather_entity.entity_list, start=index_starter_count):
print(i,str(divisor),item)
# drill
drill = int(input(question1))
drill_result = core_ds_gather_by_entity.dlist_list[drill]
print(drill_result)
filename = drill_result
output = filename[:-1]
ofile = glob(str(datascript_file_exact)+str(output),recursive=recursive_set_true)
cleaned_ofile = str(ofile).replace(replace_list_index, blank)
cleaned_ofile_master = cleaned_ofile.replace(replace_entity_title_back, blank)
global func_a
func_a = cleaned_ofile_master
cleaned_output = open(func_a, 'r')
datascript_content = cleaned_output.read()
print(datascript_content)
entity_construct()
else:
print("use 'direct_open(object)' to open an ENTITY directly")
def entity_list_drill(object=None):
"""
Inspect an entity, but don't load it into the 'entities' variable
"""
if object == None:
for i, item in enumerate(core_gather_entity.entity_list, start=index_starter_count):
print(i,str(divisor),item)
drill = int(input(question1))
drill_result = core_gather_entity.entity_list[drill]
print(drill_result)
else:
print("use 'direct_view(object)' to view and ENTITY directly")
## Create additional utility functions
def direct_open(object=None):
"""
Use the 'direct_open(object)' convention to directly open an entity.
The object will become loaded in the 'entities_direct' variable
dlist_drill() will load all entity files into the 'entities' variable
"""
def construct(object):
"""
Build file sorting relationships
"""
filename = object
output = filename
print(output)
ofile = glob(str(datascript_file_exact)+str(output),recursive=recursive_set_true )
cleaned_ofile = str(ofile).replace(replace_list_index,blank)
cleaned_ofile_master = cleaned_ofile.replace(replace_entity_title_back,blank)
global func_a
func_a = cleaned_ofile_master
cleaned_output = open(func_a, 'r')
datascript_content = cleaned_output.read()
print(datascript_content)
entity_construct()
print("Located the following ENTITY: Relationship: "+object)
if object == None:
"""
Falls back to input mode if no object was specified
"""
ds_file_direct = input("Enter the verbose name of the .ds file you would like to view.")
if ds_file_direct == '':
try:
object = ds_file_direct
construct(object)
except FileNotFoundError:
print("This .ds entry wasn't found in your Entity Archive")
finally:
pass
else:
try:
object = ds_file_direct
construct(object)
except FileNotFoundError:
print("This .ds entry wasn't found in your Entity Archive")
finally:
pass
else:
"""
Put a filter for your desired fire. direct_open(object="DS5555555.ds")
try to construct(object), but if it's not found, exit. Errors pass to end of loop
"""
try:
construct(object)
except FileNotFoundError:
print("This .ds entry wasn't found in your Entity Archive")
finally:
pass
def direct_view(object=None):
"""
Use the 'direct_open(object)' convention to directly open an entity.
The object will become loaded in the 'entities_direct' variable
dlist_drill() will load all entity files into the 'entities' variable
running direct_view will allow you to import an entity for viewing directly
by providing the DS*******.ds file location
"""
def view(object):
"""
View a speicific entity
"""
filename = object
output = filename
print(output)
ofile = glob(str(datascript_file_exact)+str(output),recursive=recursive_set_true)
cleaned_ofile = str(ofile).replace(replace_list_index,blank)
cleaned_ofile_master = cleaned_ofile.replace(replace_entity_title_back,blank)
global func_a
func_a = cleaned_ofile_master
cleaned_output = open(func_a, 'r')
datascript_content = cleaned_output.read()
print(datascript_content)
print("You are Viewing the following ENTITY: "+object)
print("Note* if you would like to load the entity, use 'direct_open(object)' instead")
if object == None:
"""
Falls back to input mode if no object was specified
"""
ds_file_direct = input("Please specify the .ds object you would like to load...")
if ds_file_direct == '':
print("No file specified... try again")
else:
try:
object = ds_file_direct
direct_view(object)
except FileNotFoundError:
print("This .ds entry wasn't found in your Entity Archive")
finally:
pass
else:
"""
print(drill_result)Put a filter for your desired fire. direct_open(object="DS5555555.ds")
try to construct(object), but if it's not found, exit. Errors pass to end of loop
"""
try:
view(object)
except FileNotFoundError:
print("This .ds entry wasn't found in your Entity Archive")
finally:
pass
Return HOME