{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#SecondoMonitor -s in Secondo/bin\n", "#StartOptServer in Secondo/Optimizer\n", "#sgui in Secondo/Javagui\n", "#ISO-8859-1" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from IPython.core.debugger import set_trace\n", "import os\n", "import configparser as cfg\n", "import pandas as pd\n", "import socket\n", "import sys\n", "import time\n", "from io import *\n", "import asyncio\n", "import ast\n", "import nest_asyncio\n", "from struct import *\n", "nest_asyncio.apply()\n", "\n", "\n", "Type_dict = {\"BIN_LONGLIST\": 0,\n", " \"BIN_INTEGER\": 1,\n", " \"BIN_REAL\": 2,\n", " \"BIN_BOOLEAN\": 3,\n", " \"BIN_LONGSTRING\": 4,\n", " \"BIN_LONGSYMBOL\": 5,\n", " \"BIN_LONGTEXT\": 6,\n", " \"BIN_LIST\": 10,\n", " \"BIN_SHORTLIST\": 11,\n", " \"BIN_SHORTINT\": 12,\n", " \"BIN_BYTE\":13,\n", " \"BIN_STRING\": 14,\n", " \"BIN_SHORTSTRING\": 15,\n", " \"BIN_SYMBOL\": 16,\n", " \"BIN_SHORTSYMBOL\": 17,\n", " \"BIN_TEXT\": 18,\n", " \"BIN_SHORTTEXT\": 19,\n", " \"BIN_DOUBLE\": 20}\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "########## Nested List manipulation functions ############\n", "\n", "\n", "#returns a double quoted string for types string and text but symbols remain with single quotation mark\n", "class dbl_quot(str):\n", " def __repr__(self):\n", " return ''.join(('\"', super().__repr__()[1:-1], '\"'))\n", "\n", "#updates a specific element by value in a nested list with unknown depth and structure\n", "def change(seq, what, make):\n", " for i, item in enumerate(seq):\n", " if item == what:\n", " seq[i] = make\n", " elif type(item) == list:\n", " change(item, what, make)\n", " return seq\n", "\n", "\n", "#modifying the text types of form '****' to ****\n", "def change_textform(seq):\n", " for i, item in enumerate(seq):\n", " #if type(item) != list and seq[i] and seq[i] != '' and item[0] == \"'\" and item[1] != \"(\":\n", " # seq[i] = \"\" + item[1:]\n", " # j = i+1 \n", " # while True: \n", " \n", " # seq[i] += seq[j]\n", " # del seq[j] \n", " # \n", " # if seq[j] == \"'\":\n", " # seq[i] += \"\"\n", " # del seq[j]\n", " # break\n", " #elif:\n", " if seq[i] and seq[i] != '' and item[0] == \"'\" and item[1] == \"(\":\n", " seq[i] = \"\" + item[1:-1] + \"\"\n", " \n", " elif seq[i] and seq[i] == '':\n", " item = [\"\",seq[i+1],\"\"] \n", " seq[i] = item\n", " del seq[i+1], seq[i+1]\n", " \n", " elif seq[i] and type(item) == list:\n", " change_textform(item)\n", " \n", " return seq\n", "\n", "\n", "#flattens a nested list with unknown depth and structure to a flat list\n", "def flatten_list(lisst):\n", " lst = []\n", " for i in lisst:\n", " if isinstance (i, list):\n", " lst.extend (flatten_list (i))\n", " else:\n", " lst.append (i)\n", " return lst \n", "\n", "\n", "#returns index of first appearance of item in a nested list with unknown depth and structure\n", "def nested_index(lst, target):\n", " for index, item in enumerate(lst):\n", " if item == target:\n", " return [index]\n", " if isinstance(item, (list, tuple)):\n", " path = nested_index(item, target)\n", " if path:\n", " return [index] + path\n", " return []\n", "\n", "\n", "#indices=[]\n", "#for item in flattend_list:\n", "# indices.append(nested_index(nestedlist,item))\n", "#print(indices)\n", "\n", "#distinguishing the data types of nested list items\n", "def NL_itemsTypeCheck(NL):\n", " flattend_list = flatten_list(NL)\n", " #print (flattend_list)\n", " for item in flattend_list:\n", " \n", " #if not item.startswith('\"') and not item.startswith('') and not item.startswith('') and item != ' ' and item.lower() != 'from' and item[-1] != ':' and item[-1] != '.' and item.lower() != \"is\":\n", " if item.isdigit() or item in ('True', 'False'):\n", " try:\n", " if type(eval(item)) in(int, float, bool):\n", " change(NL, item, ast.literal_eval(item))\n", " except (ValueError, NameError):\n", " pass\n", " #print (NL)\n", " return(NL)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# config_Datei lesen\n", "class Config():\n", " \n", " def __init__(self,file=None,sec_srv=None,user=None,passw=None,sec_port=None,opt_srv=None,opt_port=None,opt_enable=None):\n", " if file and not (sec_srv and user and passw and sec_port and opt_srv and opt_port and opt_enable):\n", " self.file_name = file\n", " self.sec_srv = \"\"\n", " self.user = \"\"\n", " self.passw = \"\"\n", " self.sec_port = 0\n", " self.opt_srv = \"\"\n", " self.opt_port = 0\n", " self.opt_enable = \"\"\n", " self.startconnection = False\n", " self.cfg = cfg.ConfigParser()\n", " self.get_from_file()\n", " \n", " elif not (file and sec_srv and user and passw and sec_port and opt_srv and opt_port and opt_enable):\n", " print(\"A needed parameter is missing!\")\n", " elif not file and (sec_srv and user and passw and sec_port and opt_srv and opt_port and opt_enable):\n", " self.sec_srv = sec_srv\n", " self.user = user\n", " self.passw = passw\n", " self.sec_port = sec_port\n", " self.opt_srv = opt_srv\n", " self.opt_port = opt_port\n", " self.opt_enable = opt_enable\n", " self.startconnection = True\n", " \n", " def ConfigSectionMap(self, section):\n", " dict1 = {}\n", " options = self.cfg.options(section)\n", " for option in options:\n", " try:\n", " dict1[option] = self.cfg.get(section, option)\n", " if dict1[option] == -1:\n", " DebugPrint(\"skip: %s\" % option)\n", " except:\n", " print(\"exception on %s!\" % option)\n", " dict1[option] = None\n", " return dict1\n", "\n", " def get_from_file(self):\n", " #dict1= ConfigSectionMap\n", " if os.path.isfile(os.getcwd() + '/' + self.file_name):\n", " self.cfg.read(os.getcwd() + '/' + self.file_name)\n", " self.sec_srv = self.ConfigSectionMap(\"General\")['servername']\n", " self.sec_port = self.ConfigSectionMap(\"General\")['serverport']\n", " self.startconnection = self.ConfigSectionMap(\"General\")['start_connection']\n", " self.user = self.ConfigSectionMap(\"General\")['user']\n", " self.passw = self.ConfigSectionMap(\"General\")['passwd']\n", " self.opt_srv = self.ConfigSectionMap(\"General\")['optimizer_host']\n", " self.opt_port = self.ConfigSectionMap(\"General\")['optimizer_port']\n", " self.opt_enable = self.ConfigSectionMap(\"General\")['enable_optimizer']\n", " \n", " else:\n", " print(\"The ConfigFile not found!\") \n", "\n", " if self.opt_srv == \"\":\n", " self.opt_srv = \"localhost\"\n", " if int(self.opt_port) <= 0:\n", " self.opt_port = 1235\n", " \n", " def initialize(self):\n", " print(self.sec_srv, self.sec_port, self.startconnection, self.user, self.passw, self.opt_srv, self.opt_port, self.opt_enable)\n", " return self.sec_srv, self.sec_port, self.startconnection, self.user, self.passw, self.opt_srv, self.opt_port, self.opt_enable \n", "\n", "conn = Config(\"gui.cfg\")\n", "params = conn.initialize()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "#Connecting to SecondoServer with acyncio method\n", "\n", "class Secondo():\n", " def __init__(self, server, port):\n", " \n", " self.server = server\n", " self.port = port\n", " self.initialized = False\n", " self.message = \"\\n\" + params[3] + \"\\n\" + params[4] + \"\\n\" + \"\\n\"\n", " self.conn = None\n", " self.reader = None\n", " self.writer = None\n", " self.opendb = \"\"\n", " self.binLists = True\n", " loop = asyncio.get_event_loop()\n", " loop.run_until_complete(self.connect())\n", " \n", " async def connect(self):\n", " self.conn = asyncio.open_connection(self.server, int(self.port))\n", " self.reader, self.writer = await asyncio.wait_for(self.conn, timeout = 10)\n", " \n", " while True:\n", " line = await self.reader.readline()\n", " if not line:\n", " print(\"No response from Server!\")\n", " break\n", " line = line.decode()\n", " if line != '\\n':\n", " print(f'Received: {line!r}')\n", " break\n", " \n", " if line == '\\n':\n", " print(f'Received: {line}')\n", " print(self.message)\n", " \n", " self.writer.write(self.message.encode())\n", " await self.writer.drain()\n", " \n", " while True:\n", " line = await self.reader.readline()\n", " \n", " if not line:\n", " print(\"Connection to Server failed!\")\n", " break\n", "\n", " line = line.decode()\n", " if line == '\\n':\n", " print(f'Received: {line!r}')\n", " while True:\n", " line = await self.reader.readline()\n", " \n", " if not line:\n", " print(\"Connection to Server failed!\")\n", " line = line.decode()\n", " \n", " if line == '\\n':\n", " print(f'Received: {line!r}')\n", " self.initialized = True\n", " print(\"Connection To Secondo Server established...\")\n", " break\n", " \n", " elif line == '\\n':\n", " print(f'Received: {line!r}')\n", " else:\n", " print(f'Received: {line!r}')\n", " break\n", " \n", " \n", " break \n", " print(self.initialized)\n", " \n", " def get_sec_conn(self):\n", " if self.conn:\n", " return self.conn\n", " else:\n", " print(\"Connection Error!\")\n", " return None\n", " \n", " def get_sec_streams(self):\n", " if self.reader and self.writer:\n", " return self.reader, self.writer\n", " else:\n", " print(\"Connection Error, Streams not connected!\")\n", " return None\n", " \n", " async def reopen_db(self):\n", " if len(self.get_opendb()) == 0:\n", " return False\n", " db = self.get_opendb()\n", " reader, writer = self.get_sec_streams()\n", " res = await exec_command(reader, writer, \"close database\")\n", " \n", " if (int.from_bytes(res[0][0], byteorder='big')) != 0:\n", " return False\n", " res = await exec_command(reader, writer, \"open database\" + db)\n", " return (int.from_bytes(res[0][0], byteorder='big')) == 0\n", " \n", " def get_opendb(self):\n", " return self.opendb\n", " \n", " def set_opendb(self, db):\n", " self.opendb = db\n", " \n", " def get_binLists(self):\n", " return self.binLists\n", " \n", " def set_binLists(self, val):\n", " self.binLists = val\n", " \n", "\n", "sec = Secondo(params[0], params[1])\n", "reader, writer = sec.get_sec_streams()\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# connecting to OptimizerServer using asyncio methode\n", "\n", "class Optimizer():\n", " \n", " def __init__(self, server, port, enable):\n", " \n", " self.server = server\n", " self.port = port\n", " self.enable = enable\n", " self.initialized = False\n", " self.conn = None\n", " self.optreader = None\n", " self.optwriter = None\n", " loop = asyncio.get_event_loop()\n", " loop.run_until_complete(self.connect())\n", " \n", " \n", " async def connect(self):\n", " \n", " if self.enable != \"true\":\n", " print(\"Optimizer Server cant not be contacted according to configurations.\")\n", " return\n", " elif self.initialized:\n", " print(\"Connection to Optimizer Server is already stablished.\")\n", " return\n", " elif self.enable == \"true\" and self.initialized == False:\n", " print(\"Starting the connection...\")\n", " self.conn = asyncio.open_connection(self.server, int(self.port))\n", " self.optreader, self.optwriter = await asyncio.wait_for(self.conn, timeout = 10)\n", " \n", " message = \"\\n\"\n", " while True:\n", " self.optwriter.write(message.encode(\"utf-8\"))\n", " await self.optwriter.drain()\n", " \n", " print(\"Message sent to optserver...\")\n", " print(message.encode(\"utf-8\"))\n", " \n", " print(\"wants to read from optserver...\")\n", " line = await self.optreader.readline()\n", " print(\"read from optserver...\")\n", " \n", " line = line.decode(\"utf-8\")\n", " print(f'Received: {line!r}')\n", " \n", " if not line:\n", " print(\"No response from Server!\")\n", " self.initialized = False\n", " break\n", " \n", " if line != \"\\n\":\n", " print(f'Received wrong data: {line!r}')\n", " self.initialized = False\n", " break\n", " \n", " if line == \"\\n\":\n", " #print(f'Received: {line!r}')\n", " self.initialized = True\n", " print(\"Connection to Optimizer stablished...\")\n", " break\n", " \n", " print(self.initialized)\n", " \n", " def get_opt_conn(self):\n", " if self.conn:\n", " return self.conn\n", " else:\n", " print(\"Connection Error!\")\n", " return None\n", " \n", " def get_opt_streams(self):\n", " if self.optreader and self.optwriter:\n", " return self.optreader, self.optwriter\n", " else:\n", " print(\"Connection Error, Streams not connected!\")\n", " return None\n", " \n", " \n", "opt = Optimizer(params[5], params[6], params[7])\n", "optreader, optwriter = opt.get_opt_streams()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "#Parsing Secondo_NestedList to a Python_Nestedlist\n", "\n", "\n", "from pyparsing import *\n", "import pprint\n", "import re\n", "\n", "def NLparse(content, ftype): #ftype = True: file as argument, ftype = False: string as argument\n", " #ident = Word(alphas, alphanums + \"-_.\")\n", " #number = Word('-'+'+'+nums+'.')\n", " #nestedParens = nestedExpr('(', ')', content=enclosed) \n", " enclosed = Forward()\n", " nestedParens = nestedExpr('(', ')')\n", " enclosed << (OneOrMore(nestedParens))\n", " if ftype:\n", " try:\n", " with open(content, encoding=\"utf8\", errors='ignore') as content_file:\n", " \n", " content = content_file.read()\n", " if not content.startswith('('):\n", " content = '(' + content + ')'\n", " NLresult = enclosed.parseString(content).asList()\n", " \n", " except (ParseException, ParseFatalException) as err:\n", " NLresult = []\n", " print(err)\n", " print(\"Invalid syntax at line {}, column {}: '{}': {}.\".format(err.lineno,err.column,err.markInputline(),err.msg))\n", "\n", " else:\n", " \n", " try:\n", " if not content.startswith('('):\n", " content = '(' + content + ')'\n", " NLresult = enclosed.parseString(content).asList()\n", " \n", " except (ParseException, ParseFatalException) as err:\n", " NLresult = []\n", " print(err)\n", " print(\"Invalid syntax at line {}, column {}: '{}': {}.\".format(err.lineno,err.column,err.markInputline(),err.msg))\n", " \n", " print(NLresult[0]) \n", " NLresult = change_textform(NLresult[0])\n", " NLresult = NL_itemsTypeCheck(NLresult)\n", " print('result of NLParser: ')\n", " print(NLresult)\n", " return NLresult\n", "\n", " \n", " \n", "#converts the python nested list to a string with nested parenthesis\n", "def NLtoString(NL):\n", " strresult = pprint.pformat(NL)\n", " replacements = {\"[\": \"(\", \"]\": \")\", \"'\": \"\"}\n", " strresult = \"\".join([replacements.get(c, c) for c in strresult])\n", " #print(strresult)\n", " return strresult\n", "\n", "\n", "#NL=NLparse(\"opt.txt\", True)\n", "#NL=NLparse(\"(restore database berlintest from berlintest)\", False)\n", "#print(NL)\n", "\n", "\n", "\n", "\n", "#returns the type of the given item of NestedList\n", "def AtomType(item):\n", " \n", " if isinstance(item, str) and item[0] == '\"':\n", " return \"string\"\n", " if isinstance(item, str) and item[0] != '\"' and item[0] != '<':\n", " return \"symbol\"\n", " if isinstance(item, int):\n", " return \"integer\"\n", " if isinstance(item, float):\n", " return \"real\"\n", " if isinstance(item, bool):\n", " return \"boolean\"\n", " if (isinstance(item, list) and item[0] == \"\") or (isinstance(item, str) and item.startswith('')):\n", " return \"text\"\n", " if isinstance(item, list):\n", " return \"list\"\n", " \n", " \n", "def BinaryType(item):\n", " itemType = AtomType(item)\n", " if itemType == \"string\":\n", " length = len(item)\n", " if length < 256:\n", " return 'BIN_SHORTSTRING'\n", " if length < 65536:\n", " return 'BIN_STRING'\n", " return 'BIN_LONGSTRING'\n", " if itemType == \"symbol\":\n", " length = len(item)\n", " if length < 256:\n", " return 'BIN_SHORTSYMBOL'\n", " if length < 65536:\n", " return 'BIN_SYMBOL'\n", " return 'BIN_LONGSYMBOL'\n", " if itemType == \"integer\":\n", " if item in range(-128, 128):\n", " return 'BIN_BYTE'\n", " if item in range(-32768, 32768):\n", " return 'BIN_SHORTINT'\n", " return 'BIN_INTEGER'\n", " if itemType == \"real\":\n", " return 'BIN_DOUBLE'\n", " if itemType == \"boolean\":\n", " return 'BIN_BOOLEAN'\n", " if itemType == \"text\":\n", " length = len(item)\n", " if length < 256:\n", " return 'BIN_SHORTTEXT'\n", " if length < 65536:\n", " return 'BIN_TEXT'\n", " return 'BIN_LONGTEXT'\n", " if itemType == \"list\":\n", " length = len(item)\n", " if length < 256:\n", " return 'BIN_SHORTLIST'\n", " if length < 65536:\n", " return 'BIN_LIST'\n", " return 'BIN_LONGLIST'\n", " \n", " \n", "async def WriteListToBinary(writer, NL):\n", " if not writer:\n", " return False\n", " sig = bytearray(\"bnl\")\n", " major = pack('>H', 1)\n", " minor = pack('>H', 2)\n", " writer.write(sig)\n", " writer.write(major)\n", " writer.write(minor)\n", " await writer.drain()\n", " ok = await ListToBinary(writer, NL)\n", " await writer.drain()\n", " return ok\n", " \n", " \n", "async def ListToBinary(writer, NL):\n", " btype = BinaryType(NL)\n", " typeCode = None\n", " for typ, val in Type_dict.items():\n", " if btype == typ:\n", " typeCode = val\n", " break\n", " \n", " if btype == 'BIN_BOOLEAN':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " val = 1 if NL esle 0\n", " writer.write(pack('?', val)[0])\n", " await writer.drain()\n", " return True\n", " \n", " if btype == 'BIN_INTEGER':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(pack('>I', NL))\n", " await writer.drain()\n", " return True\n", " if btype == 'BIN_SHORTINT':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(pack('>H', NL))\n", " await writer.drain()\n", " return True\n", " if btype == 'BIN_BYTE':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(NL.to_bytes(1, byteorder='big'))\n", " await writer.drain()\n", " return True\n", " \n", " if btype == 'BIN_REAL':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(pack('>f', NL)[0])\n", " await writer.drain()\n", " return True\n", " \n", " if btype == 'BIN_DOUBLE':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(pack('>d', NL)[0])\n", " await writer.drain()\n", " return True\n", " \n", " if btype == 'BIN_DOUBLE':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(pack('>d', NL)[0])\n", " await writer.drain()\n", " return True\n", " \n", " if btype == 'BIN_SHORTSTRING':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(len(NL[0]).to_bytes(1, byteorder='big'))\n", " writer.write(NL[0].encode())\n", " await writer.drain()\n", " return True\n", "\n", " if btype == 'BIN_STRING':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(pack('>H', len(NL[0]))\n", " writer.write(NL[0].encode())\n", " await writer.drain()\n", " return True\n", " \n", " if btype == 'BIN_LONGSTRING':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(pack('>I', len(NL[0]))\n", " writer.write(NL[0].encode())\n", " await writer.drain()\n", " return True\n", " \n", " if btype == 'BIN_SHORTSYMBOL':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(len(NL[0]).to_bytes(1, byteorder='big'))\n", " writer.write(NL[0].encode())\n", " await writer.drain()\n", " return True\n", "\n", " if btype == 'BIN_SYMBOL':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(pack('>H', len(NL[0]))\n", " writer.write(NL[0].encode())\n", " await writer.drain()\n", " return True\n", " \n", " if btype == 'BIN_LONGSYMBOL':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(pack('>I', len(NL[0]))\n", " writer.write(NL[0].encode())\n", " await writer.drain()\n", " return True\n", "\n", " if btype == 'BIN_SHORTTEXT':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(len(NL[0]).to_bytes(1, byteorder='big'))\n", " writer.write(NL[0].encode())\n", " await writer.drain()\n", " return True\n", " \n", " if btype == 'BIN_TEXT':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(pack('>H', len(NL[0]))\n", " writer.write(NL[0].encode())\n", " await writer.drain()\n", " return True\n", "\n", " if btype == 'BIN_LONGTEXT':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(pack('>I', len(NL[0]))\n", " writer.write(NL[0].encode())\n", " await writer.drain()\n", " return True\n", "\n", " if btype == 'BIN_SHORTLIST':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(len(NL).to_bytes(1, byteorder='big'))\n", " for i in range(len(NL)):\n", " res = await ListToBinary(writer, NL[i])\n", " if not res:\n", " return False\n", " await writer.drain()\n", " return True\n", " \n", " if btype == 'BIN_LIST':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(pack('>H', len(NL))\n", " for i in range(len(NL)):\n", " res = await ListToBinary(writer, NL[i])\n", " if not res:\n", " return False\n", " await writer.drain()\n", " return True\n", " \n", " if btype == 'BIN_LONGLIST':\n", " writer.write(typeCode.to_bytes(1, byteorder='big'))\n", " writer.write(pack('>I', len(NL))\n", " for i in range(len(NL)):\n", " res = await ListToBinary(writer, NL[i])\n", " if not res:\n", " return False\n", " await writer.drain()\n", " return True\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "async def readBinaryRecord():\n", " print(\"begin of readBinaryRecord().\")\n", " Type = await reader.readexactly(1)\n", " Type = int.from_bytes(Type, byteorder='big', signed=False)\n", " print(\"first byte read in readBinaryRecord()\")\n", " print(Type)\n", " \n", " Type_dict = {\"BIN_LONGLIST\": 0,\n", " \"BIN_INTEGER\": 1,\n", " \"BIN_REAL\": 2,\n", " \"BIN_BOOLEAN\": 3,\n", " \"BIN_LONGSTRING\": 4,\n", " \"BIN_LONGSYMBOL\": 5,\n", " \"BIN_LONGTEXT\": 6,\n", " \"BIN_LIST\": 10,\n", " \"BIN_SHORTLIST\": 11,\n", " \"BIN_SHORTINT\": 12,\n", " \"BIN_BYTE\":13,\n", " \"BIN_STRING\": 14,\n", " \"BIN_SHORTSTRING\": 15,\n", " \"BIN_SYMBOL\": 16,\n", " \"BIN_SHORTSYMBOL\": 17,\n", " \"BIN_TEXT\": 18,\n", " \"BIN_SHORTTEXT\": 19,\n", " \"BIN_DOUBLE\": 20}\n", " typeCode = None\n", " for typ, val in Type_dict.items():\n", " if val == Type:\n", " typeCode = val\n", " break\n", " \n", " if typeCode == 0: #LongList\n", " print(\"LongList\")\n", " LE = []\n", " barray = bytearray(4) ####readInt()\n", " barray = await reader.readexactly(4)\n", " print(barray)\n", " assert (int.from_bytes(barray, byteorder = 'big', signed = True)) >= 0, \"Invalid Integer-Byte read from Stream!\"\n", " r = unpack('>I',barray)[0]\n", " ####\n", " if r == 0:\n", " return LE\n", " else:\n", " for i in range(r):\n", " LE.append(await readBinaryRecord())\n", " return LE\n", " \n", " elif typeCode == 1:\n", " ####readInt()\n", " \n", " print(\"Int\")\n", " barray = bytearray(4)\n", " barray = await reader.readexactly(4)\n", " print(barray)\n", " assert (int.from_bytes(barray, byteorder = 'big', signed = True)) >= 0, \"Invalid Integer-Byte read from Stream!\"\n", " r = unpack('>I',barray)[0]\n", " ####\n", " print(\"int: \")\n", " print(r)\n", " return r \n", " \n", " elif typeCode == 2:\n", " ####readReal()\n", " print(\"Real\")\n", " barray = bytearray(4)\n", " barray = await reader.readexactly(4)\n", " print(barray)\n", " assert (int.from_bytes(barray, order = 'big', signed = True)) >= 0, \"Invalid Real-Byte read from Stream!\"\n", " r = unpack('>f',barray)[0]\n", " ####\n", " print(\"real: \")\n", " print(r)\n", " return r \n", " \n", " elif typeCode == 3:\n", " ####readBool()\n", " print(\"Bool\")\n", " barray = bytearray(1)\n", " barray = await reader.readexactly(1)\n", " print(barray)\n", " assert (int.from_bytes(barray, order = 'big', signed = True)) >= 0, \"Invalid Boolean-Byte read from Stream!\"\n", " r = unpack('?',barray)[0]\n", " ####\n", " print(\"bool: \")\n", " print(r)\n", " return r\n", " \n", " \n", " elif typeCode == 4:\n", " ####readLongString()\n", " print(\"LongString\")\n", " barray = bytearray(4) #readInt()\n", " barray = await reader.readexactly(4)\n", " print(barray)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid Int-Byte read from Stream!\"\n", " len = unpack('>I',barray)[0]\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " print(barray)\n", " #for b in barray:\n", " #assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid LongSymbol-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " print(\"long string: \")\n", " print(barray.decode())\n", " return '\"' + barray.decode() + '\"'\n", " \n", " elif typeCode == 5:\n", " ####readLongSymbol()\n", " print(\"LongSymbol\")\n", " barray = bytearray(4) #readInt()\n", " barray = await reader.readexactly(4)\n", " print(barray)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid Int-Byte read from Stream!\"\n", " len = unpack('>I',barray)[0]\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " print(barray)\n", " #for b in barray:\n", " #assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid LongSymbol-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " print(\"long symbol: \")\n", " print(barray.decode())\n", " return barray.decode()\n", " \n", " elif typeCode == 6:\n", " ####readLongText()\n", " print(\"LongText\")\n", " barray = bytearray(4) #readInt()\n", " barray = await reader.readexactly(4)\n", " print(barray)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid Int-Byte read from Stream!\"\n", " len = unpack('>I',barray)[0]\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " print(barray)\n", " #for b in barray:\n", " #assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid LongText-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " print(\"Long Text: \")\n", " txt = barray.decode()\n", " if not txt.startswith(''):\n", " txt = '\"' + txt + '\"'\n", " \n", " print(txt)\n", " return txt\n", " \n", " elif typeCode == 10: #List\n", " print(\"List\")\n", " LE = []\n", " ####readShort()\n", " barray = bytearray(2)\n", " barray = await reader.readexactly(2)\n", " print(barray)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid ShortInt-Byte read from Stream!\"\n", " r = unpack('>H',barray)[0]\n", " ####\n", " if r == 0:\n", " return LE\n", " else:\n", " for i in range(r):\n", " LE.append(await readBinaryRecord())\n", " return LE\n", " \n", " elif typeCode == 11: #ShortList\n", " print(\"ShortList\")\n", " LE = []\n", " ####readByte()\n", " barray = bytearray(1)\n", " barray = await reader.readexactly(1)\n", " print(barray)\n", " assert (int.from_bytes(barray, byteorder = 'big', signed = True)) >= 0, \"Invalid Byte read from Stream!\"\n", " r = int.from_bytes(barray, byteorder = 'big', signed = False)\n", " ####\n", " if r == 0:\n", " return LE\n", " else:\n", " for i in range(r):\n", " LE.append(await readBinaryRecord())\n", " return LE\n", " \n", " elif typeCode == 12:\n", " ####readShort()\n", " print(\"readShort\")\n", " barray = bytearray(2)\n", " barray = await reader.readexactly(2)\n", " print(barray)\n", " #for b in barray:\n", " #assert (int.from_bytes(b, byteorder = 'big', signed = True)) >= 0, \"Invalid ShortInt-Byte read from Stream!\"\n", " r = unpack('>H',barray)[0]\n", " print(\"short: \")\n", " print(r)\n", " return r\n", " \n", " elif typeCode == 13:\n", " ####readByte()\n", " print(\"Byte\")\n", " barray = bytearray(1)\n", " barray = await reader.readexactly(1)\n", " print(barray)\n", " #assert (int.from_bytes(barray, byteorder = 'big', signed = True)) >= 0, \"Invalid Byte read from Stream!\"\n", " r = int.from_bytes(barray, byteorder = 'big', signed = True)\n", " print(\"byte: \")\n", " print(r)\n", " return r\n", " \n", " elif typeCode == 14:\n", " ####readString()\n", " print(\"String\")\n", " barray = bytearray(2)\n", " barray = await reader.readexactly(2)\n", " print(barray)\n", " for b in barray:\n", " assert (int.from_bytes(b, byteorder = 'big', signed = True)) >= 0, \"Invalid ShortInt-Byte read from Stream!\"\n", " len = unpack('>H',barray)[0]\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " print(barray)\n", " #for b in barray:\n", " #assert (int.from_bytes(b, byteorder = 'big', signed = True)) >= 0, \"Invalid String-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " print(\"string: \")\n", " print(barray.decode())\n", " return '\"' + barray.decode() + '\"'\n", " \n", " elif typeCode == 15:\n", " ####readShortString()\n", " print(\"ShortString\")\n", " barray = bytearray(1)\n", " barray = await reader.readexactly(1)\n", " print(barray)\n", " assert (int.from_bytes(barray, byteorder = 'big', signed = True)) >= 0, \"Invalid Byte read from Stream!\"\n", " len = int.from_bytes(barray, byteorder = 'big', signed = False)\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " print(barray)\n", " #for b in barray:\n", " #assert (int.from_bytes(b, byteorder = 'big', signed = True)) >= 0, \"Invalid ShortString-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " print(\"short string: \")\n", " print(barray.decode())\n", " return '\"' + barray.decode() + '\"'\n", " \n", " elif typeCode == 16:\n", " ####readSymbol()\n", " print(\"Symbol\")\n", " barray = bytearray(2) #readShort()\n", " barray = await reader.readexactly(2)\n", " print(barray)\n", " for b in barray:\n", " assert (int.from_bytes(b, byteorder = 'big', signed = True)) >= 0, \"Invalid ShortInt-Byte read from Stream!\"\n", " len = unpack('>H',barray)[0]\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " print(barray)\n", " #for b in barray:\n", " #assert (int.from_bytes(b, byteorder = 'big', signed = True)) >= 0, \"Invalid String-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " print(\"symbol: \")\n", " print(barray.decode())\n", " return barray.decode()\n", " \n", " elif typeCode == 17:\n", " ####readShortSymbol()\n", " print(\"ShortSymbol\")\n", " barray = bytearray(1) #readByte()\n", " barray = await reader.readexactly(1)\n", " print(barray)\n", " assert (int.from_bytes(barray, byteorder = 'big', signed = True)) >= 0, \"Invalid Byte read from Stream!\"\n", " len = int.from_bytes(barray, byteorder = 'big', signed = False)\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " print(barray)\n", " #for b in barray:\n", " #assert (int.from_bytes(b, byteorder = 'big', signed = True)) >= 0, \"Invalid ShortString-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " print(\"short symbol: \")\n", " print(barray.decode())\n", " return barray.decode()\n", " \n", " elif typeCode == 18:\n", " ####readText()\n", " print(\"Text\")\n", " barray = bytearray(2) #readShort()\n", " barray = await reader.readexactly(2)\n", " print(barray)\n", " #for b in barray:\n", " # assert (int.from_bytes(b, byteorder = 'big', signed = True)) >= 0, \"Invalid ShortInt-Byte read from Stream!\"\n", " len = unpack('>H',barray)[0]\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " print(barray)\n", " #for b in barray:\n", " #assert (int.from_bytes(b, byteorder = 'big', signed = True)) >= 0, \"Invalid Text-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " txt = barray.decode()\n", " if not txt.startswith(''):\n", " txt = '\"' + txt + '\"'\n", " \n", " print(txt)\n", " return txt\n", " \n", " \n", " elif typeCode == 19:\n", " ####readShortText()\n", " print(\"ShortText\")\n", " barray = bytearray(1) #readByte()\n", " barray = await reader.readexactly(1)\n", " print(barray)\n", " #assert (int.from_bytes(barray, byteorder = 'big', signed = True)) >= 0, \"Invalid Byte read from Stream!\"\n", " len = int.from_bytes(barray, byteorder = 'big', signed = False)\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " print(barray)\n", " #for b in barray:\n", " #assert (int.from_bytes(b, byteorder = 'big', signed = True)) >= 0, \"Invalid ShortText-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " print(\"short Text: \")\n", " txt = barray.decode()\n", " if not txt.startswith(''):\n", " txt = '\"' + txt + '\"'\n", " \n", " print(txt)\n", " return txt\n", " \n", " elif typeCode == 20:\n", " ####readDouble()\n", " print(\"Double\")\n", " barray = bytearray(8)\n", " barray = await reader.readexactly(8)\n", " print(barray)\n", " assert (int.from_bytes(barray, byteorder = 'big', signed = True)) >= 0, \"Invalid Double-Byte read from Stream!\"\n", " r = unpack('>d',barray)[0]\n", " ####\n", " print(\"Double: \")\n", " print(r)\n", " return r\n", "\n", "\n", "###################################################\n", "\n", "\n", "async def Receive_List(finalToken, binLists):\n", " print(\"begin of Receive_List()\")\n", " answerList = []\n", " if not binLists:\n", " result = ''\n", " count = 1\n", " while True: \n", " line = await reader.readline()\n", " print(str(count) + \"th line in Receive_List() read.\")\n", " print(line)\n", " count += 1\n", " #line = line.decode().rstrip()\n", " line = line.decode('ISO-8859-1').rstrip()\n", " if finalToken in line:\n", " #result += line + \"\\n\"\n", " result += line\n", " break\n", " if line:\n", " #result += line + \"\\n\"\n", " result += line # + ' '\n", " else:\n", " print(\"Empty Line read by Receive_List()!\")\n", " print(result)\n", " answerList= NLparse(result, False)\n", " \n", " else: #binLists = True --> readBinaryFrom(inputStream)\n", " #answer = await reader.readline()\n", " #print(answer)\n", " print(\"binarylist = True.\")\n", " line = await reader.readexactly(3)\n", " signature = line.decode().rstrip()\n", " print(\"signature: \" + signature)\n", " \n", " #readShort()\n", " barray = bytearray(2)\n", " barray = await reader.readexactly(2)\n", " major = unpack('>H',barray)\n", " major = major[0]\n", " print(\"major: \" + str(major))\n", " \n", " \n", " barray = await reader.readexactly(2)\n", " minor = unpack('>H',barray)\n", " minor = minor[0]\n", " print(\"minor: \" + str(minor))\n", " \n", " \n", " if signature != \"bnl\":\n", " print(signatur + \" is wrong Signatur!\")\n", " return None\n", " if (major != 1 or (minor != 0 and minor != 1 and minor != 2)):\n", " print(str(major) + \".\" + str(minor) + \" is wrong Version Number!\")\n", " return None\n", " print(\"readBinaryRecords() is called.\")\n", " answerList = await readBinaryRecord()\n", " ####check finalToken here#####\n", " line = await reader.readline()\n", " print(\"last line in Receive_list() in binarylist mode: \")\n", " print(line.decode())\n", " \n", " print('last result in Receive_List(): ')\n", " print(answerList)\n", " return answerList\n", "\n", "\n", "\n", "\n", "\n", "####################################################\n", "\n", "\n", "async def Receive_Response(sec):\n", " reader, writer = sec.get_sec_streams()\n", " if not (reader and writer):\n", " print('Connection to SecondoServer reset!')\n", " return None\n", " print(\"We r in Receive_Response() before reading.\")\n", " line = await reader.readline()\n", " print(\"first line read in Receive_Response(): \")\n", " print(line)\n", " line = line.decode().rstrip()\n", " if not line:\n", " print(\"Network Error!\")\n", " return None\n", " count = 2\n", " while line == \"\":\n", " print(\"Receive_List() called.\")\n", " Message_List = await Receive_List(\"\", sec.get_binLists())\n", " if not Message_List:\n", " print(\"Error- MessageList is empty!\")\n", " return None\n", " \n", " line = await reader.readline()\n", " print(str(count) + \"th line read in Receive_Response(): \")\n", " print(line)\n", " count += 1\n", " line = line.decode().rstrip()\n", " if not line:\n", " print(\"Network Error!\")\n", " return None\n", " \n", " if line == \"\":\n", " line = await reader.readline()\n", " print(\"first line read in Receive_Response() after : \")\n", " print(line)\n", " line = line.decode().rstrip()\n", " if line:\n", " line = await reader.readline()\n", " print(\"second line read in Receive_Response() after : \")\n", " print(line)\n", " line = line.decode().rstrip()\n", " \n", " else:\n", " print(\"Error reading SecondoError!\")\n", " return None\n", " return None\n", " \n", " if line != \"\":\n", " print(\"Protocol Error!\")\n", " return None\n", " \n", " print(\"Receive_List() is called.\")\n", " answerList = await Receive_List(\"\", sec.get_binLists())\n", " print('result in Receive_Response():')\n", " print(answerList)\n", " if not answerList:\n", " return None\n", " \n", " #Error_Code = answerList[0]\n", " #Error_Pos = answerList[1]\n", " #Error_Message = answerList[2]\n", " #Result_List = answerList[3]\n", " #return Result_List\n", " return answerList" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "#Command format: restore [database] database_name from \n", "async def restore_command(sec, command):\n", " reader, writer = sec.get_sec_streams()\n", " if reader and writer:\n", " if not command.startswith('(') :\n", " command = '(' + command.strip() + ')'\n", " \n", " CmdNL = NLparse(command, False)\n", " print(CmdNL)\n", " length = len(CmdNL)\n", " if length != 4 and length != 5:\n", " print(\"restore_command(): command lenght wrong!\") \n", " return None\n", " if length == 5: #restore database\n", " if CmdNL[0] != 'restore' or CmdNL[1] != 'database' or CmdNL[3] != 'from':\n", " print(\"restore_command: command wrong\") \n", " return None\n", " name = CmdNL[2]\n", " filename = CmdNL[4]\n", " database = True\n", " if length == 4:\n", " if CmdNL[0] != 'restore' or CmdNL[2] != 'from':\n", " print(\"restore_command: command wrong\") \n", " return None\n", " name = CmdNL[1]\n", " filename = CmdNL[3]\n", " database = False\n", " \n", " typ = AtomType(filename)\n", " if not typ in (\"string\", \"symbol\", \"text\"):\n", " print(\"restore_command: filename wrong\") \n", " return None\n", " \n", " \n", " tag = \"DbRestore\" if database else \"ObjectRestore\"\n", " message = \"<\" + tag + \">\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", " \n", "\n", " message = name + \"\\n\"\n", " print(\"name:\")\n", " print(name)\n", " message = message.encode()\n", " writer.write(message)\n", " \n", " \n", " message = \"\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", " \n", " with open(filename, \"rb\") as f:\n", " #with open(filename, \"rb\", encoding=\"utf8\", errors='ignore') as f: \n", " byte = f.read(1)\n", " size = 0\n", " while byte:\n", " size += 1\n", " byte = f.read(1)\n", " print(str(size)) \n", " \n", " \n", " #flen1 = os.path.getsize(f.name)\n", " #flen = len(file)\n", " message = str(size) + \"\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", " await writer.drain()\n", " \n", " \n", " #reading contents of file via io.stream\n", " stream = open(filename, 'rb')\n", " message = stream.read()\n", " stream.close()\n", " writer.write(message)\n", " \n", " #with open(filename, \"rb\") as f:\n", " #, encoding=\"utf8\", errors='ignore'\n", " #message = f.read()\n", " #writer.write(message)\n", " \n", " message = \"\\n\"\n", " message = message.encode()\n", " #print(message)\n", " writer.write(message)\n", " \n", " message = \"\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", " await writer.drain()\n", " #print(message)\n", " \n", " #### secondoresponse- ReceiveResponse\n", " print(\"calling Receive_Response()\")\n", " resp = await Receive_Response(sec)\n", " #print(resp)\n", " if resp[0] == 0:\n", " print('restore was seccessful!')\n", " else:\n", " print('Error occured during restore!')\n", " \n", " return resp\n", " \n", " else:\n", " print(\"reader and writer Sockets null\")\n", " return None\n", " \n", " \n", "\n", "#Command format: save database/ to \n", "async def save_command(sec, command):\n", " reader, writer = sec.get_sec_streams()\n", " if reader and writer:\n", " if not command.startswith('(') :\n", " command = '(' + command.strip() + ')'\n", " \n", " CmdNL = NLparse(command, False)\n", " print(CmdNL)\n", " \n", " \n", " if CmdNL[0] != 'save' or CmdNL[2] != 'to':\n", " print(\"save_command: command wrong\") \n", " return None\n", " obj_name = CmdNL[1]\n", " filename = CmdNL[3]\n", " \n", " typ = AtomType(filename)\n", " if not typ in (\"string\", \"symbol\", \"text\"):\n", " print(\"save_command: filename wrong\") \n", " return None\n", " \n", " if obj_name == \"database\":\n", " writer.write(\"\\n\".encode())\n", " else:\n", " writer.write(\"\\n\".encode())\n", " writer.write((obj_name + \"\\n\").encode())\n", " writer.write(\"\\n\".encode())\n", " \n", " await writer.drain()\n", " resp = await Receive_Response(sec)\n", " if resp:\n", " if resp[0] == 0:\n", " stream = open(filename, 'w')\n", " stream.writelines(resp[3])\n", " stream.close()\n", " print('Save Successful!')\n", " return resp\n", " else:\n", " return None\n", " else:\n", " print(\"reader and writer Sockets null\")\n", " return None\n", "\n", "\n", "async def general_command(sec, command):\n", " reader, writer = sec.get_sec_streams()\n", " if reader and writer:\n", " \n", " if 'pysend[' in command.lower():\n", " \n", " async def handle_RecFromSec(reader, writer):\n", " \n", " list = os.listdir(os.getcwd() + '/tmp')\n", " \n", " try:\n", " f = open('tmp/send' + str(len(list) + 1), 'w')\n", " count = 0\n", " while True:\n", " line = await reader.readline()\n", " f.write(line.decode())\n", " \n", " #line = base64.b64decode(line)\n", " #f.write(base64.b64decode(line) + '\\n')\n", " \n", " count +=1\n", " if 'quit' in line.decode():\n", " break\n", " except IOError:\n", " print(\"Error writing to file!\")\n", " finally:\n", " f.close()\n", " \n", " print('Stream of {} Tuples received from SecondoServer.'.format(str(count)))\n", " \n", " \n", " idx1 = command.find('pysend[') + 7\n", " idx2 = command.find(']', int(idx1))\n", " port = int(command[idx1:idx2])\n", " print('port:')\n", " print(port)\n", " loop = asyncio.get_event_loop()\n", " loop.create_task(asyncio.start_server(handle_RecFromSec, '127.0.0.1', port))\n", " \n", " if 'pyreceive[' in command.lower():\n", " \n", " async def handle_send2Sec(reader, writer):\n", " #try:\n", " #file = input(\"Enter the file name containing the tuples:\\n\")\n", " #except EOFError as e:\n", " #print(\"Error getting the file name!\")\n", " \n", " try:\n", " f = open('tmp/send19', 'r')\n", " lines = f.readlines()\n", " except IOError:\n", " print(\"No such file!\")\n", " finally:\n", " f.close()\n", " \n", " count = 0\n", " for line in lines:\n", " if not line.startswith('(stream') and not line.startswith('quit'):\n", " print (line)\n", " writer.write(line.encode())\n", " await writer.drain()\n", " count +=1\n", " \n", " print('Stream of {} Tuples sent to SecondoServer.'.format(str(count)))\n", " writer.close()\n", " \n", " idx1 = command.find('pyreceive[') + 10\n", " idx2 = command.find(']', int(idx1))\n", " port = int(command[idx1:idx2])\n", " loop = asyncio.get_event_loop()\n", " loop.create_task(asyncio.start_server(handle_send2Sec, '127.0.0.1', port))\n", "\n", " \n", " cmdLevel = str(0 if ((command.lstrip()).lower()).startswith('(') else 1)\n", " \n", " message = \"\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", " \n", " cmdLevel += \"\\n\"\n", " message = cmdLevel.encode()\n", " writer.write(message)\n", " \n", " message = command\n", " message = message.encode()\n", " writer.write(message)\n", "\n", " message = \"\\n\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", " await writer.drain()\n", " \n", " print(\"in general_command writing finished, if Pysend and receive start server, run Receive_Response(sec)\")\n", " result = await Receive_Response(sec)\n", " print(\"in general_command writing finished Receive_Response(sec)\")\n", " return result\n", " else:\n", " print(\"reader and writer Sockets null\")\n", " return None\n", "\n", "\n", " \n", "async def opt_comm_exec(opt, query, db_name, flag):\n", " optrdr, optwrt = opt.get_opt_streams()\n", " if optreader is None or optwriter is None:\n", " print(\"Connection Error!\")\n", " return None\n", " else:\n", " \n", " #execute query\n", " if flag:\n", " optwrt.write(b'\\n')\n", " \n", " #optimize query\n", " else: \n", " optwrt.write(b'\\n')\n", " \n", " optwrt.write(b'\\n')\n", " db_name += \"\\n\"\n", " optwrt.write(db_name.encode())\n", " \n", " optwrt.write(b'\\n')\n", " \n", " optwrt.write(b'\\n')\n", " query += \"\\n\"\n", " optwrt.write(query.encode())\n", " \n", " optwrt.write(b'\\n')\n", " \n", " if flag:\n", " optwrt.write(b'\\n')\n", " \n", " else: \n", " optwrt.write(b'\\n')\n", " \n", " await optwrt.drain()\n", " \n", " res = await optrdr.readline()\n", " res = res.decode()\n", " print(\"first response from OptServer: \")\n", " print(res)\n", " if not res:\n", " print(\"Connection to Optimizer is broken!\")\n", " return None\n", " if res != \"\\n\":\n", " print(\"Protocol Error!\")\n", " return None\n", " #res = await optrdr.readuntil(b'')\n", " res = await optrdr.readline()\n", " print(\"second response from OptServer: \")\n", " print(res)\n", " res = res.decode()\n", " if not res:\n", " print(\"Connection to Optimizer is broken!\")\n", " return None\n", " opt_res = \"\"\n", " count = 3\n", " while res != \"\\n\":\n", " if flag:\n", " opt_res += res\n", " else:\n", " opt_res += res.replace(\" \\n\", \"\")\n", " res = await optrdr.readline()\n", " print(str(count) + \"th response from OptServer: \")\n", " print(res)\n", " res = res.decode()\n", " count += 1\n", " if not res:\n", " print(\"Connection to Optimizer is broken!\")\n", " return None\n", " \n", " if opt_res == \"\":\n", " print(\"Optimization failed!\")\n", " return None\n", " print(opt_res)\n", " return opt_res\n", " \n", " \n", " \n", "async def send_to_optimizer(sec, opt, command):\n", " update_comm = False\n", " catalog_comm = False\n", " select_comm = True\n", " if command.startswith('sql') or command.startswith('sql\\n'):\n", " update_comm = True\n", " elif \"insert into \" in command:\n", " update_comm = True\n", " elif \"delete from \" in command:\n", " update_comm = True\n", " elif \"update \" in command:\n", " update_comm = True\n", " elif \"create table\" in command:\n", " update_comm = True\n", " catalog_comm = True\n", " select_comm = False\n", " elif \"drop\" in command:\n", " update_comm = True\n", " catalog_comm = True\n", " select_comm = False \n", " elif \"select\" in command:\n", " update_comm = True\n", " \n", " if update_comm:\n", " if len(sec.get_opendb()) == 0:\n", " print(\"No open Database!\")\n", " return None\n", " \n", " flg = False\n", " db = sec.get_opendb()\n", " qry = command\n", " if not opt:\n", " print('Connection to OptimizerServer reset!')\n", " return None\n", " opt_res = await opt_comm_exec(opt, qry, db, flg)\n", " \n", " if opt_res:\n", " if catalog_comm:\n", " db = await sec.reopen_db()\n", " if db:\n", " print(\"Reopenning the DataBase after Catalog update, was seccessful!\")\n", " else: \n", " print(\"Reopenning the DataBase after Catalog update, was Not seccessful!\")\n", " return None\n", " \n", " elif select_comm:\n", " return \"query \" + opt_res\n", " \n", " return opt_res\n", " \n", " \n", " else:\n", " print(\"Optimization failed!\")\n", " return None\n", " \n", " else:\n", " return command\n", " \n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": false }, "outputs": [], "source": [ "async def exec_command(sec, opt, command):\n", " if not sec:\n", " print('Connection to ScondoServer reset!')\n", " return None\n", " if not opt:\n", " print('Connection to OptimizerServer reset!')\n", " return None\n", " \n", " reader, writer = sec.get_sec_streams()\n", " \n", " command = await send_to_optimizer(sec, opt, command)\n", " print('command after optimization: ')\n", " print(command)\n", " #handling restore-command\n", " if command and (((command.lstrip()).lower()).startswith('(restore') or ((command.lstrip()).lower()).startswith('( restore') or ((command.lstrip()).lower()).startswith('restore')):\n", " result = await restore_command(sec, command)\n", " if not result:\n", " print('Command execution returned an empty list!') \n", " return None\n", " if sec.get_binLists():\n", " if result[0] == 0 :\n", " print('Command execution was successful!')\n", " else:\n", " print('Command execution returned an Error!')\n", " else:\n", " print('The reselt in Text-Mode:')\n", " if result[0] == 0:\n", " print('Command execution was successful!')\n", " else:\n", " print('Command execution returned an Error!')\n", " return result\n", " \n", " #handliung save-command\n", " if command and (((command.lstrip()).lower()).startswith('(save') or ((command.lstrip()).lower()).startswith('( save') or ((command.lstrip()).lower()).startswith('save')):\n", " result = await save_command(sec, command)\n", " if not result:\n", " print('Command execution returned an empty list!') \n", " return None\n", " if sec.get_binLists():\n", " if result[0] == 0 :\n", " print('Command execution was successful!')\n", " else:\n", " print('Command execution returned an Error!')\n", " else:\n", " print('The reselt in Text-Mode:')\n", " if result[0] == 0:\n", " print('Command execution was successful!')\n", " else:\n", " print('Command execution returned an Error!')\n", " return result\n", " \n", " #handling general-commands\n", " \n", " result = await general_command(sec, command)\n", " \n", " if not result:\n", " print('Command execution returned an empty list!')\n", " return None\n", " \n", " if sec.get_binLists():\n", " if result[0] == 0 :\n", " print('Command execution was successful!')\n", " else:\n", " print('Command execution returned an Error!')\n", " else:\n", " print('The reselt in Text-Mode:')\n", " if result[0] == 0:\n", " print('Command execution was successful!')\n", " else:\n", " print('Command execution returned an Error!')\n", " \n", " if sec.get_binLists() and result[0] == 0 and \"open database \" in command:\n", " sec.set_opendb(command[14:])\n", " elif not sec.get_binLists() and result[0] == 0 and \"open database \" in command:\n", " sec.set_opendb(command[14:])\n", " \n", " print(result)\n", " return result\n", " \n", " \n", "#com = 'open database berlintest'\n", "#com = 'sql select * from staedte where bev > 100000'\n", "#com = 'query plz feed head[7] pysend[30000] consume'\n", "\n", "#com = 'let ReviewsSchema1 = [const rel(tuple([Plz: int, Ort: string])) value ()]'\n", "#com = 'let Reviews = ReviewsSchema1 pyreceive[30000] consume'\n", "\n", "#com = 'query [ const rel(tuple([Plz: int, Ort: string])) value() ] pyreceive[30000] count'\n", "#com = 'query [ const rel(tuple([Plz: int, Ort: string])) value() ] pyreceive[30000] consume'\n", "\n", "#com = 'query plz feed head[10] count'\n", "#com = 'query Staedte feed filter[(.Bev > 100000)] count'\n", "#com = 'query Staedte feed filter[(.Bev > 100000)] {1} consume'\n", "#com = 'list algebras'\n", "#com = 'list algebra PyStreamAlgebra'\n", "#com = 'restore database berlintest from berlintest'\n", "#com = 'close database'\n", "\n", "\n", "\n", "rs = await exec_command(sec, opt, com)\n", "print(rs)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#await asyncio.wait_for(reader.readline(),timeout=2)\n", "\n", "#import io\n", "#stream = io.open(\"berlintest\", \"rb\")\n", "#print(stream.read())\n", "\n", "\n", "#finding the index of an item in a list\n", "#index(...)\n", "#L.index(value, [start, [stop]]) -> integer -- return first index of value\n", "\n", "#\n", "#i = iter(lisst)\n", "#first = next(i)\n", "#rest = list(i)\n", "\n", "\n", "#recursive approach to itterate a nested list of unknown depth and structure\n", "#def traverse(o, tree_types=(list, tuple)):\n", "# if isinstance(o, tree_types):\n", "# for value in o:\n", "# for subvalue in traverse(value, tree_types):\n", "# yield subvalue\n", "# else:\n", "# yield o\n", "\n", "#a = number of inner lists\n", "#lst = [[] for _ in range(a)]\n", "\n", "#int(b'0x16'.decode(), 16)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.4" } }, "nbformat": 4, "nbformat_minor": 2 }