# -*- coding: utf-8 -*-
"""
COPYRIGHT (C) 2020-2023 NEW ENTITY OPERATIONS INC. ALL RIGHTS RESERVED
INSTANCE: build
MODIFIED: 2023/05/10
OVERVIEW: Run this program with python build.py to assemble the NOVASTORE
data-structure
NOVASTORE will assemble itself based on the inputs provided to the
system-wide instance in TRINE.
When you need to specific endpoints, you can retrieve them from CORE
by retrieving them by a valid key i.e. CoreRetrieve("B03A")
This will also set the progams default build parameters
"""
__version__ = "1.0.0"
__author__ = "New Entity Operations Inc."
__copyright__ = "Copyright (C) 2020-2023 New Entity Operations Inc."
__credits__ = ["New Entity Operations Inc.", "Ryan McKenna"]
__email__ = "Operator@NewEntityOperations.com"
__license__ = "New Entity License"
__maintainer__ = "Ryan McKenna"
__status__ = "Production"
## imports: custom
from MODE.facilities import (ACTIVE_TOKEN, ACTIVE_TOKEN_BUCKET,
ALREADY_CRAWED_BUCKET, CRAW_BUCKET_NOW, Cure, DictAssemblyLine,
DictPath, EXHAUSTED_TOKEN, FINAL_BUCKET, FORMATTED_TOKEN_BUCKET,
glob, IGNORE_BUCKET, NESTED_NODE, oFo, path, Path, PATH_NOW,
SLUG_BUCKET, SysPath, TOKENIZE_NOW, TokenSlug)
from FILTER.regex import Extractor as eX
from MODE.debug_build import DEBUG_BUILD
## establish the system path
path_project = path.abspath(Cure.muted)
SysPath.insert(0, path_project)
## Instance construction prompt
DEBUG_BUILD.default_node_assembler(NODE_MAIN=path_project)
## local instance assembly hook Add the root path
## Provide a DEFAULT_PATH for the _CRAWER
DEFAULT_PATH_CRAWER = path_project+TokenSlug.glob_location
START_SLUG = TokenSlug.Start_Slug+str(TokenSlug.Start_Number)
FINAL_BUCKET.append(">>>>"+path_project+"<<<"+START_SLUG)
## NodeAssembler
class NodeAssembler:
"""
Define the node structure for your NOVASTORE instance
"""
def build(Node=Cure.muted, PathFull=Cure.muted):
"""
Tokenizers start with <<<, and are in-line
Root structures start with >>>>
Sub-directories start with >>>
Files start with >>
Transmution-linkers start with <
Each layer switches between a number and a letter and if need be, will repeat
"""
COUNTER_FILE = 0
COUNTER_NESTED_NODE = 0
COUNTER_ANCHOR = 0
COUNTER_SUB_NODE = 0
if Node==Cure.muted:
DEBUG_BUILD.node_anchor_empty()
else:
if PathFull==Cure.muted:
DEBUG_BUILD.node_location_unavailable()
else:
DEBUG_BUILD.building_now(LOCATION=Node)
NODE_TOKENIZER_BUCKET = []
structure_node_reader = open("structure.ds", oFo.read)
for i in structure_node_reader:
DEBUG_BUILD.node_evaluation(LOCATION=i)
if i.startswith('>>>>') == True:
if COUNTER_ANCHOR == 1:
DEBUG_BUILD.building_failed_duplicate_nodes()
else:
DEBUG_BUILD.anchor_set(LOCATION=i)
COUNTER_ANCHOR = COUNTER_ANCHOR+1
ROOT_SLUG = eX.FINDER_ROOT.findall(i)
LOCATION_SLUG = eX.TOKEN_FINDER.findall(i)
NODE_TOKENIZER_BUCKET.append(
"Slug:Node-> "+ROOT_SLUG[0]+" : Location "+LOCATION_SLUG[0])
DictAssemblyLine['node_slug'] = [LOCATION_SLUG[0], ROOT_SLUG[0]]
elif i.startswith('>>>') == True:
DEBUG_BUILD.anchor_nested(LOCATION=i)
COUNTER_SUB_NODE = COUNTER_SUB_NODE+1
SUB_NODE_SLUG = eX.FINDER_SUB_NODE.findall(i)
LOCATION_SLUG = eX.TOKEN_FINDER.findall(i)
NODE_TOKENIZER_BUCKET.append(
"Slug:Node-> "+SUB_NODE_SLUG[0]+" : Location "+LOCATION_SLUG[0])
DictAssemblyLine['node_slug'].append([LOCATION_SLUG[0], SUB_NODE_SLUG[0]])
elif i.startswith('<') == True:
DEBUG_BUILD.anchor_exhausted(LOCATION=i)
COUNTER_NESTED_NODE = COUNTER_NESTED_NODE+1
FINDER_NESTED_NODE_SLUG = eX.FINDER_NESTED_NODE.findall(i)
FINDER_NESTED_NODE_NAME_SLUG = eX.FINDER_NESTED_NODE_NAME.findall(i)
LOCATION_SLUG = eX.TOKEN_FINDER.findall(i)
## Get the second instance of LOCATION_SLUG
try:
NODE_TOKENIZER_BUCKET.append(
"Slug:Node-> "+FINDER_NESTED_NODE_SLUG[0]+\
" : Location "+"<<"+LOCATION_SLUG[0])
DictAssemblyLine['node_slug'].append(
[LOCATION_SLUG[0], "Belongs to->"+\
str(FINDER_NESTED_NODE_SLUG[0]), "Name-> "+\
FINDER_NESTED_NODE_NAME_SLUG[0]])
except IndexError as e:
DEBUG_BUILD.error_syntax()
## At the bottom so we don't catch any of the other conditions
elif i.startswith('>>') == True:
DEBUG_BUILD.endpoint(ENDPOINT=i)
COUNTER_FILE = COUNTER_FILE+1
FILE_SLUG = eX.FINDER_FILE.findall(i)
FILE_NAME_SLUG = eX.FINDER_NESTED_NODE.findall(i)
LOCATION_SLUG = eX.TOKEN_FINDER.findall(i)
NODE_TOKENIZER_BUCKET.append(
"Slug:File-> "+FILE_SLUG[0]+" : Location "+LOCATION_SLUG[0])
DictAssemblyLine['node_slug'].append(
[LOCATION_SLUG[0], "Belongs to->"+\
str(FILE_SLUG[0]), "Name-> "+FILE_NAME_SLUG[1]+"."+FILE_NAME_SLUG[2]])
else:
pass
DEBUG_BUILD.node_tokenizer_assembled(
ANCHOR=DictAssemblyLine['node_slug'][0:2],
BUCKET=NODE_TOKENIZER_BUCKET[:],
DICTIONARY=DictAssemblyLine)
DictCombCounter = 2
try:
while True:
z = DictAssemblyLine['node_slug'][DictCombCounter]
DictCombCounter = DictCombCounter+1
DEBUG_BUILD.node_tokenizer_added_value(VALUE=z)
except IndexError as e:
DEBUG_BUILD.node_tokenizer_exhausted(error=e)
## PATH
class PATH:
PATH_BUCKET = [path_project]
def path_logic(path_dest=Cure.muted):
"""
clear the current PATH_BUCKET and determine what path to assemble next
"""
PATH.PATH_BUCKET.clear()
if path_dest==Cure.muted:
## if no path is supplied fall back to the provided default
DEBUG_BUILD.path_invalid()
z = glob(DEFAULT_PATH_CRAWER)
for i in z:
PATH.PATH_BUCKET.append(i)
else:
## if a path is provided in path_dest, utilize it
z = glob(path_dest)
for i in z:
PATH.PATH_BUCKET.append(i)
def construct_path(i=Cure.muted, TOKEN=Cure.muted):
if TokenSlug.Sub_Level_Counter == 0:
## The first run executes this, or being passed a default
##FinalBucket.make() value the run must also be checked for overloading
if TokenSlug.OVERLOADED >= 1:
## If the path is overloaded, use the overloaded path
PATH.path_logic(path_dest=i+TokenSlug.glob_nested)
PATH_NOW.clear()
PATH_NOW.append(path_project)
ACTIVE_TOKEN.clear()
ACTIVE_TOKEN.append(TokenSlug.Start_Slug+str(TokenSlug.Start_Number))
DictPath[Path(path_project)] = ACTIVE_TOKEN[0]
else:
## If the path isn't overloaded, use the default path
PATH.path_logic(path_dest=DEFAULT_PATH_CRAWER)
PATH_NOW.clear()
PATH_NOW.append(path_project)
ACTIVE_TOKEN.clear()
ACTIVE_TOKEN.append(TokenSlug.Start_Slug+str(TokenSlug.Start_Number))
DictPath[Path(path_project)] = ACTIVE_TOKEN[0]
else:
## All subsequent runs will have a Sub_Level_Counter of >= 1
ALREADY_CRAWED_BUCKET.append(i)
PATH_NOW.clear()
PATH_NOW.append(i)
DictPath[Path(i)] = TOKEN
PATH.path_logic(path_dest=i+TokenSlug.glob_nested)
PATH.PATH_BUCKET
## SlugMaker
class SlugMaker:
def construct():
## On the first run 'A' and '0'
SLUG_BUCKET.append(TokenSlug.Start_Slug+str(TokenSlug.Start_Number))
## Increment the Start_Number, which will reset on each provided TOKEN
TokenSlug.Start_Number = TokenSlug.Start_Number + 1
## Clone the PATH.PATH_BUCKET into the updated SLUG_BUCKET
for i in PATH.PATH_BUCKET:
SLUG_BUCKET.append(i)
def IncrementSlugCounter():
## On the first run, this will go from 0->1
TokenSlug.SLUG_COUNTER = TokenSlug.SLUG_COUNTER+1
DEBUG_BUILD.increment_token_counter(
FROM=TokenSlug.SLUG_COUNTER, TO=TokenSlug.SLUG_COUNTER+1)
## The tree logic goes here
def LocationSlug():
## Make the 'deep crawl' logic
try:
## Condition 1: If the token has no more available slugs
if TokenSlug.available_slugs[TokenSlug.SLUG_COUNTER] == \
TokenSlug.available_slugs[-1]:
DEBUG_BUILD.node_recursion_item_nested(ITEM=TokenSlug.Sub_Level_Counter)
## Add a ladder incrementer
#TokenSlug.LADDER = TokenSlug.LADDER + 1
TokenSlug.Start_Slug = \
str(TokenSlug.LADDER)+TokenSlug.available_slugs[TokenSlug.SLUG_COUNTER]
## Condition 2, the slugs are root level complex directories
else:
DEBUG_BUILD.node_recursion_item_anchor(ITEM=TokenSlug.Sub_Level_Counter)
#TokenSlug.LADDER = TokenSlug.LADDER + 1
TokenSlug.Start_Slug = TokenSlug.available_slugs[TokenSlug.SLUG_COUNTER]
except IndexError as e:
## Condition 3, the slugs are in a sub-node
if "list index out of range" in str(e):
## If the node is the anchor and complex,
if TokenSlug.Sub_Level_Counter == 0:
DEBUG_BUILD.node_recursion_item_sub(ITEM=TokenSlug.Sub_Level_Counter)
TokenSlug.Start_Slug = \
str(TokenSlug.LADDER)+\
TokenSlug.available_slugs[TokenSlug.SLUG_COUNTER]+\
TokenSlug.available_slugs[TokenSlug.SLUG_COUNTER]
TokenSlug.Sub_Level_Counter = TokenSlug.LADDER+TokenSlug.Sub_Level_Counter
else:
DEBUG_BUILD.node_recursion_item_sub(ITEM=TokenSlug.Sub_Level_Counter)
TokenSlug.Sub_Level_Counter = TokenSlug.Sub_Level_Counter + 1
## In this case, we always reset the index
## Reset the counter and add an index location
TokenSlug.SLUG_COUNTER = 0
## Condition 4, undefined
else:
DEBUG_BUILD.error_recorded(error=e)
def build():
## Reload the SLUG_BUCKET index
SlugMaker.construct()
## Advance through the Slug structure levels
SlugMaker.IncrementSlugCounter()
## Tokenizer
class Tokenizer:
def evaluate():
## Construct the SlugMaker.build() routine
SlugMaker.build()
## Utilize the SLUG_BUCKET to determine the index
for i in SLUG_BUCKET:
if str(i) == str(SLUG_BUCKET[0]):
DEBUG_BUILD.token_index(INDEX=i)
## Look for matches, and if a match occurs overwrite the name of the INDEX
for M in EXHAUSTED_TOKEN:
DEBUG_BUILD.mode_value_exhausted(EXHAUSTED=M)
if M == SLUG_BUCKET[0]:
DEBUG_BUILD.confirmed_exhausted_mode(VALUE=SLUG_BUCKET[0])
OLD_SLUG = SLUG_BUCKET[0]
SLUG_BUCKET.clear()
SLUG_BUCKET.append(OLD_SLUG+"A"+str(1))
DEBUG_BUILD.remade_slug_bucket(WITH=SLUG_BUCKET[0])
#TokenSlug.ALTER_INDEX_COUNTER = 1
else:
DEBUG_BUILD.path_mismatch()
else:
DEBUG_BUILD.generated_index_mapper(MAPPER=SLUG_BUCKET[0])
## Determine if the i value is a file,
## and if it is append the correct syntax
if path.isfile(i):
DEBUG_BUILD.located_endpoint(ENDPOINT=i)
## Always TOKENIZE the instnace
TOKENIZE_NOW.append(i)
FORMATTED_TOKEN_BUCKET.append(
">><"+SLUG_BUCKET[0]+">"+i+"<<<"+\
str(SLUG_BUCKET[0])+str(TokenSlug.Start_Number))
## Determine if the i value is a Node,
## and if it is append the correct syntax
else:
DEBUG_BUILD.located_node(NODE=i)
IGNORE_CHECK = Path(i)
CHECKED_SLUG = IGNORE_CHECK.stem
DEBUG_BUILD.node_stem(STEM=CHECKED_SLUG)
## Determine if the i value is restricted Node and if it is, exclude it
if CHECKED_SLUG in IGNORE_BUCKET or "/"+CHECKED_SLUG+"/" in i:
DEBUG_BUILD.node_ignored_or_already_checked()
## If the Node is no retricted, determine if it is a
## anchor level or Sub level instance. Then, apply the correct syntax
else:
## Always TOKENIZE the instnace
TOKENIZE_NOW.append(i)
## Add the instance to the CRAW_BUCKET_NOW
CRAW_BUCKET_NOW.append(i)
## Add the instance to the NESTED_NODE
NESTED_NODE.append(i)
## If the Token is the same as the initial TOKEN,
## we are dealing with a root level instance
if TokenSlug.Start_Slug == TokenSlug.available_slugs[0]:
FORMATTED_TOKEN_BUCKET.append(
">>><"+SLUG_BUCKET[0]+">"+i+"<<<"+str(SLUG_BUCKET[0]))
## If the Token is not the same as the initial TOKEN,
## we are dealing with a sub level instance
else:
FORMATTED_TOKEN_BUCKET.append(
"<"+SLUG_BUCKET[0]+">"+i+"<<<"+str(SLUG_BUCKET[0]))
## Exhaust the token when it's a node instance
EXHAUSTED_TOKEN.append(SLUG_BUCKET[0])
DEBUG_BUILD.tokenizing_now()
## Format the TOKEN to the desired result
for i in TOKENIZE_NOW:
DEBUG_BUILD.token_now(
BELONGS_TO=SLUG_BUCKET[0],
ITEM=i, NUMBER=TokenSlug.Start_Number,
TOKEN=SLUG_BUCKET[0])
DEBUG_BUILD.token_now_formatted(
TOKEN_FORMATTED=\
FORMATTED_TOKEN_BUCKET[TokenSlug.FORMATTED_TOKEN_NUMBER]+\
str(TokenSlug.Start_Number))
SLUG = \
FORMATTED_TOKEN_BUCKET[TokenSlug.FORMATTED_TOKEN_NUMBER]+\
str(TokenSlug.Start_Number)
DEBUG_BUILD.token_formatted(
TOKEN_FORMATTED=SLUG_BUCKET[0]+str(TokenSlug.Start_Number))
TokenSlug.FORMATTED_TOKEN_NUMBER = TokenSlug.FORMATTED_TOKEN_NUMBER + 1
FINAL_BUCKET.append(SLUG)
DEBUG_BUILD.finalizing_slug(VALUE=SLUG)
TokenSlug.Start_Number = TokenSlug.Start_Number + 1
## Add to the TokenSlug.Sub_Level_Counter to
## indicate additional sturcture levels
TokenSlug.Sub_Level_Counter = TokenSlug.Sub_Level_Counter + 1
TOKENIZE_NOW.clear()
## Make the location slug
SlugMaker.LocationSlug()
## Clear past values and reset ques
SLUG_BUCKET.clear()
FORMATTED_TOKEN_BUCKET.clear()
TokenSlug.Start_Number = 0
TokenSlug.FORMATTED_TOKEN_NUMBER = 0
DEBUG_BUILD.node_evaluation_crawler(TREE=CRAW_BUCKET_NOW[:])
## Operate
class Operate:
def RUN_CRAW_PARSER():
## Determine the length of the current CRAW_BUCKET_
g = len(CRAW_BUCKET_NOW)
DEBUG_BUILD.node_craw_length(LENGTH=g)
## For each item in the CRAW_BUCKET_
for i in CRAW_BUCKET_NOW:
## Skip all slugs that have already been crawed
if i in ALREADY_CRAWED_BUCKET:
DEBUG_BUILD.node_craw_already_took_place()
## Continue to all uncrawed slugs
else:
## For all valid items, set the CURRENT_PARSE_BUCKET
CURRENT_PARSE_BUCKET = i
## Reset the ACTIVE_TOKEN_BUCKET
ACTIVE_TOKEN_BUCKET.clear()
## Set the ACTIVE_TOKEN_BUCKET parameters to the matched DictPath values
for k, v in DictPath.items():
DEBUG_BUILD.evaluating_dictionary(
KEY=k, VALUE=v,
PATH_K=Path(k), PATH_NOW=Path(PATH_NOW[0]))
## If the Path from the match is valid, you're on a valid craw-ready path
if str(Path(k)) == str(Path(PATH_NOW[0])):
z = str(v)
ACTIVE_TOKEN_BUCKET.append(v)
DEBUG_BUILD.path_match(MATCH=z)
else:
DEBUG_BUILD.path_mismatch()
## Once a valid ACTIVE_TOKEN_BUCKET has been built,
## you can eliminate the old lookup value
try:
## Pop at the DELIMITER
POPPED = CRAW_BUCKET_NOW.pop(TokenSlug.DELIMITER)
## Reconstruct the path with the current valid item
construct_path(i=CURRENT_PARSE_BUCKET, TOKEN=ACTIVE_TOKEN_BUCKET[0])
DEBUG_BUILD.path_popped(PATH=POPPED)
## Perform the Operation functions for the current valies
Operate.RUN_EXTENDED_PATH_LOGIC()
Operate.RUN_CRAW_PARSER()
## When there are no values left to operate on, exhaust the function
except IndexError as e:
DEBUG_BUILD.path_exhausted()
## Determine which types of data the corresponding TOKEN is linked to
def RUN_EXTENDED_PATH_LOGIC():
Tokenizer.evaluate()
## FinalBukcet
class FinalBucket:
"""
When you want to run the program, utilize the runner command->
FinalBucket.make()
"""
def make(path_overloaded=Cure.muted):
"""
First run is by construct_path(i="", TOKEN=ACTIVE_TOKEN[0]) with values:
i="", TOKEN="A0"
Allow for overloading by using FinalBucket.make(path="/YOUR_PATH/HERE")
"""
if path_overloaded==Cure.muted:
DEBUG_BUILD.path_not_overloaded()
## Default to the anchor node
construct_path(i=Cure.muted, TOKEN=ACTIVE_TOKEN_BUCKET[0])
else:
## Pass an overloaded node-start location
DEBUG_BUILD.path_overloaded(PATH=path_overloaded)
## Increment the OVERLOADED counter
TokenSlug.OVERLOADED = TokenSlug.OVERLOADED + 1
construct_path(i=path_overloaded, TOKEN=ACTIVE_TOKEN_BUCKET[0])
## Use the operations command to determine the
## EXTENDED_PATH_LOGIC and build the CRAW_PARSER
Operate.RUN_EXTENDED_PATH_LOGIC()
Operate.RUN_CRAW_PARSER()
## Reset the parameters
TokenSlug.DELIMITER = 0
CRAW_BUCKET_NOW.clear()
DEBUG_BUILD.tokenizing_bucket_final(
TOKENIZED_BUCKET_FINAL=FINAL_BUCKET[:])
DEBUG_BUILD.tokenized_output(FINALIZED_BUCKET=FINAL_BUCKET)
## View the exhausted TOKENS
DEBUG_BUILD.tokenized_output_exhausted_tokens(
EXHAUSTED_TOKENS=EXHAUSTED_TOKEN[:])
## Sort the file-system
ALREADY_CRAWED_BUCKET.sort()
DEBUG_BUILD.build_summary(INSTANCE=ALREADY_CRAWED_BUCKET)
def Flush_FINAL_BUCKET():
FINAL_BUCKET.clear()
## Program Runner
## Overloaded
#FinalBucket.make(path_overloaded="/NOVASTORE")
## Default
FinalBucket.make()
## Optionally clear the final bucket ->
## Useful if you don't want to concat two runs together
#Flush_FINAL_BUCKET()
