{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# SecondoMonitor -s in Secondo/bin\n", "#StartOptServer in Secondo/Optimizer\n", "#sgui in Secondo/Javagui\n", "\n", "#\\x00 = NULL\n", "#\\x04 = EOT (End of transmission)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from IPython.core.debugger import set_trace\n", "import os\n", "import configparser as cfg\n", "import pandas as pd\n", "import socket\n", "import sys\n", "import time\n", "from io import BytesIO\n", "from struct import *\n", "import asyncio\n", "import ast\n", "import nest_asyncio\n", "nest_asyncio.apply()\n", "binLists = True" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "########## Nested List manipulation functions ############\n", "\n", "#updates a specific element by value in a nested list with unknown depth and structure\n", "def change(seq, what, make):\n", " for i, item in enumerate(seq):\n", " if item == what:\n", " seq[i] = make\n", " elif type(item) == list:\n", " change(item, what, make)\n", " return seq\n", "\n", "\n", "#modifying the text types of form '****' to ****\n", "def change_textform(seq):\n", " for i, item in enumerate(seq):\n", " if seq[i] and seq[i] != '' and item[0] == \"'\" and item[1] != \"(\":\n", " seq[i] = \"\" + item[1:]\n", " j = i+1 \n", " while True: \n", " \n", " seq[i] += seq[j]\n", " del seq[j] \n", " \n", " if seq[j] == \"'\":\n", " seq[i] += \"\"\n", " del seq[j]\n", " break\n", " \n", " elif seq[i] and seq[i] != '' and item[0] == \"'\" and item[1] == \"(\":\n", " seq[i] = \"\" + item[1:-1] + \"\"\n", " \n", " elif seq[i] and seq[i] == '':\n", " item = [\"\",seq[i+1],\"\"] \n", " seq[i] = item\n", " del seq[i+1], seq[i+1]\n", " \n", " elif seq[i] and type(item) == list:\n", " change_textform(item)\n", " \n", " return seq\n", "\n", "\n", "#flattens a nested list with unknown depth and structure to a flat list\n", "def flatten_list(lisst):\n", " lst = []\n", " for i in lisst:\n", " if isinstance (i, list):\n", " lst.extend (flatten_list (i))\n", " else:\n", " lst.append (i)\n", " return lst \n", "\n", "\n", "#returns index of first appearance of item in a nested list with unknown depth and structure\n", "def nested_index(lst, target):\n", " for index, item in enumerate(lst):\n", " if item == target:\n", " return [index]\n", " if isinstance(item, (list, tuple)):\n", " path = nested_index(item, target)\n", " if path:\n", " return [index] + path\n", " return []\n", "\n", "\n", "#indices=[]\n", "#for item in flattend_list:\n", "# indices.append(nested_index(nestedlist,item))\n", "#print(indices)\n", "\n", "#distinguishing the data types of nested list items\n", "def NL_itemsTypeCheck(NL):\n", " flattend_list = flatten_list(NL)\n", " #print (flattend_list)\n", " for item in flattend_list:\n", " \n", " #if not item.startswith('\"') and not item.startswith('') and not item.startswith('') and item != ' ' and item.lower() != 'from' and item[-1] != ':' and item[-1] != '.' and item.lower() != \"is\":\n", " if item.isdigit() or item in ('True', 'False'):\n", " try:\n", " if type(eval(item)) in(int, float, bool):\n", " change(NL, item, ast.literal_eval(item))\n", " except (ValueError, NameError):\n", " pass\n", " #print (NL)\n", " return(NL)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# config_Datei lesen\n", "\n", "class conn():\n", " \n", " def __init__(self,file=None,sec_srv=None,user=None,passw=None,sec_port=None,opt_srv=None,opt_port=None,opt_enable=None):\n", " if file and not (sec_srv and user and passw and sec_port and opt_srv and opt_port and opt_enable):\n", " self.file_name = file\n", " self.sec_srv = \"\"\n", " self.user = \"\"\n", " self.passw = \"\"\n", " self.sec_port = 0\n", " self.opt_srv = \"\"\n", " self.opt_port = 0\n", " self.opt_enable = \"\"\n", " self.startconnection = False\n", " self.Config = cfg.ConfigParser()\n", " self.get_from_file()\n", " \n", " elif not (file and sec_srv and user and passw and sec_port and opt_srv and opt_port and opt_enable):\n", " print(\"A needed parameter is missing!\")\n", " elif not file and (sec_srv and user and passw and sec_port and opt_srv and opt_port and opt_enable):\n", " self.sec_srv = sec_srv\n", " self.user = user\n", " self.passw = passw\n", " self.sec_port = sec_port\n", " self.opt_srv = opt_srv\n", " self.opt_port = opt_port\n", " self.opt_enable = opt_enable\n", " self.startconnection = True\n", " \n", " def ConfigSectionMap(self, section):\n", " dict1 = {}\n", " options = self.Config.options(section)\n", " for option in options:\n", " try:\n", " dict1[option] = self.Config.get(section, option)\n", " if dict1[option] == -1:\n", " DebugPrint(\"skip: %s\" % option)\n", " except:\n", " print(\"exception on %s!\" % option)\n", " dict1[option] = None\n", " return dict1\n", "\n", " def get_from_file(self):\n", " #dict1= ConfigSectionMap\n", " if os.path.isfile(os.getcwd() + '/' + self.file_name):\n", " self.Config.read(os.getcwd() + '/' + self.file_name)\n", " self.sec_srv = self.ConfigSectionMap(\"General\")['servername']\n", " self.sec_port = self.ConfigSectionMap(\"General\")['serverport']\n", " self.startconnection = self.ConfigSectionMap(\"General\")['start_connection']\n", " self.user = self.ConfigSectionMap(\"General\")['user']\n", " self.passw = self.ConfigSectionMap(\"General\")['passwd']\n", " self.opt_srv = self.ConfigSectionMap(\"General\")['optimizer_host']\n", " self.opt_port = self.ConfigSectionMap(\"General\")['optimizer_port']\n", " self.opt_enable = self.ConfigSectionMap(\"General\")['enable_optimizer']\n", " \n", " else:\n", " print(\"The ConfigFile not found!\") \n", "\n", " if self.opt_srv == \"\":\n", " self.opt_srv = \"localhost\"\n", " if int(self.opt_port) <= 0:\n", " self.opt_port = 1235\n", " \n", " def initialize(self):\n", " print(self.sec_srv, self.sec_port, self.startconnection, self.user, self.passw, self.opt_srv, self.opt_port, self.opt_enable)\n", " return self.sec_srv, self.sec_port, self.startconnection, self.user, self.passw, self.opt_srv, self.opt_port, self.opt_enable \n", "\n", "conn1 = conn(\"gui.cfg\")\n", "params = conn1.initialize()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "#Connecting to SecondoServer with acyncio method\n", "\n", "class Secondo():\n", " def __init__(self, server, port):\n", " self.server = server\n", " self.port = port\n", " #self.loop = asyncLoop\n", " self.initialized = False\n", " self.message = \"\\n\" + params[3] + \"\\n\" + params[4] + \"\\n\" + \"\\n\"\n", " self.conn = None\n", " self.reader = None\n", " self.writer = None\n", " loop = asyncio.get_event_loop()\n", " loop.run_until_complete(self.connect())\n", " #loop.close()\n", " #await connect()\n", "\n", " async def connect(self):\n", " conn = asyncio.open_connection(self.server, int(self.port))\n", " if conn:\n", " self.conn = conn\n", " reader, writer = await asyncio.wait_for(conn, timeout = 120)\n", " if reader and writer:\n", " self.reader, self.writer = reader, writer\n", " #reader, writer = await asyncio.open_connection(self.server, int(self.port))\n", " \n", " while True:\n", " line = await reader.readline()\n", " if not line:\n", " print(\"No response from Server!\")\n", " self.initialized = False\n", " break\n", " line = line.decode()\n", " #.rstrip()\n", " if line != '\\n':\n", " print(f'Received: {line!r}')\n", " self.initialized = False\n", " break\n", " \n", " if line == '\\n':\n", " print(f'Received: {line}')\n", " print(self.message)\n", " \n", " writer.write(self.message.encode())\n", " await writer.drain()\n", " \n", " while True:\n", " line = await reader.readline()\n", " \n", " if not line:\n", " print(\"Connection to Server failed!\")\n", " self.initialized = False\n", " break\n", "\n", " line = line.decode()\n", " if line == '\\n':\n", " print(f'Received: {line!r}')\n", " while True:\n", " line = await reader.readline()\n", " \n", " if not line:\n", " print(\"Connection to Server failed!\")\n", " self.initialized = False\n", " break\n", " line = line.decode()\n", " \n", " if line == '\\n':\n", " print(f'Received: {line!r}')\n", " self.initialized = True\n", " print(\"Connection To Secondo Server established...\")\n", " #return reader, writer\n", " break\n", " \n", " elif line == '\\n':\n", " print(f'Received: {line!r}')\n", " self.initialized = False\n", " break\n", " else:\n", " print(f'Received: {line!r}')\n", " self.initialized = False\n", " break\n", " \n", " \n", " break \n", " print(self.initialized)\n", " \n", " def get_Sec_conn(self):\n", " if self.conn:\n", " return self.conn\n", " else:\n", " return None\n", " \n", " def get_sec_streams(self):\n", " if self.reader and self.writer:\n", " return self.reader, self.writer\n", " else:\n", " None\n", " \n", "\n", "#message = \"\\n\" + user + \"\\n\" + passw + \"\\n\" + \"\\n\"\n", "sec = Secondo(params[0], params[1])\n", "reader, writer = sec.get_sec_streams()\n", "#reader, writer = async_loop.run_until_complete(sec.connect())\n", "#async_loop.run_forever()\n", "\n", "#cr = sec.connect()\n", "#loop = asyncio.get_event_loop()\n", "#reader, writer = loop.run_until_complete(cr)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# connecting to OptimizerServer using asyncio methode\n", "\n", "class Optimizer():\n", " \n", " def __init__(self, server, port, enable):\n", " self.server = server\n", " self.port = port\n", " self.enable = enable\n", " self.initialized = False\n", " \n", " #async def tcp_opt_client\n", " async def connect(self):\n", " \n", " if self.enable != \"true\":\n", " print(\"Optimizer Server cant not be contacted according to configurations.\")\n", " elif self.initialized:\n", " print(\"Connection to Optimizer Server is already stablished.\")\n", " elif self.enable == \"true\" and self.initialized == False:\n", " print(\"Starting the connection...\")\n", " optreader, optwriter = await asyncio.open_connection(self.server, int(self.port))\n", " message = \"\\n\"\n", " \n", " while True:\n", " optwriter.write(message.encode(\"utf-8\"))\n", " await optwriter.drain()\n", " \n", " print(\"Message sent to optserver...\")\n", " print(message.encode(\"utf-8\"))\n", " \n", " print(\"wants to read from optserver...\")\n", " line = await optreader.readline()\n", " print(\"read from optserver...\")\n", " \n", " line = line.decode(\"utf-8\")\n", " print(f'Received: {line!r}')\n", " \n", " if not line:\n", " print(\"No response from Server!\")\n", " self.initialized = False\n", " break\n", " \n", " if line != \"\\n\":\n", " print(f'Received wrong data: {line!r}')\n", " self.initialized = False\n", " break\n", " \n", " if line == \"\\n\":\n", " \n", " print(f'Received: {line!r}')\n", " self.initialized = True\n", " print(\"Connection to Optimizer stablished...\")\n", " return optreader, optwriter\n", " \n", " print(self.initialized)\n", " \n", " \n", "opt = Optimizer(params[5], params[6], params[7])\n", "optreader, optwriter = await opt.connect()\n", "\n", "#cr = opt.connect()\n", "#loop = asyncio.get_event_loop()\n", "#optreader, optwriter = loop.run_until_complete(cr)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "#Parsing Secondo_NestedList to a Python_Nestedlist\n", "\n", "\n", "from pyparsing import *\n", "import pprint\n", "import re\n", "\n", "def NLparse(content, ftype): #ftype = True: file as argument, ftype = False: string as argument\n", " #ident = Word(alphas, alphanums + \"-_.\")\n", " #number = Word('-'+'+'+nums+'.')\n", " #nestedParens = nestedExpr('(', ')', content=enclosed) \n", " enclosed = Forward()\n", " nestedParens = nestedExpr('(', ')')\n", " enclosed << (OneOrMore(nestedParens))\n", " if ftype:\n", " try:\n", " with open(content, encoding=\"utf8\", errors='ignore') as content_file:\n", " \n", " content = content_file.read()\n", " NLresult = enclosed.parseString(content).asList()\n", " #print(NLresult)\n", " #return NLresult\n", " \n", " except (ParseException, ParseFatalException) as err:\n", " NLresult = []\n", " print(err)\n", " print(\"Invalid syntax at line {}, column {}: '{}': {}.\".format(err.lineno,err.column,err.markInputline(),err.msg))\n", "\n", " if not ftype:\n", " \n", " try:\n", " NLresult = enclosed.parseString(content).asList()\n", " #print(NLresult)\n", " #return NLresult\n", " \n", " except (ParseException, ParseFatalException) as err:\n", " NLresult = []\n", " print(err)\n", " print(\"Invalid syntax at line {}, column {}: '{}': {}.\".format(err.lineno,err.column,err.markInputline(),err.msg))\n", "\n", " \n", " NLresult = change_textform(NLresult)\n", " NLresult = NL_itemsTypeCheck(NLresult)\n", " return NLresult\n", "\n", " \n", " \n", "#converts the python nested list to a string with nested parenthesis\n", "def NLtoString(NL):\n", " strresult = pprint.pformat(NL)\n", " replacements = {\"[\": \"(\", \"]\": \")\", \"'\": \"\"}\n", " strresult = \"\".join([replacements.get(c, c) for c in strresult])\n", " #print(strresult)\n", " return strresult\n", "NL=NLparse(\"opt.txt\", True)\n", "#NL=NLparse(\"(restore database berlintest from berlintest)\", False)\n", "print(NL)\n", "\n", "\n", "\n", "binLists = False\n", "ftype = False\n", "\n", "#returns the type of the given item of NestedList\n", "def AtomType(item):\n", " \n", " if isinstance(item, str) and item[0] == '\"':\n", " return \"string\"\n", " if isinstance(item, str) and item[0] != '\"' and item[0] != '<':\n", " return \"symbol\"\n", " if isinstance(item, int):\n", " return \"integer\"\n", " if isinstance(item, float):\n", " return \"real\"\n", " if isinstance(item, bool):\n", " return \"boolean\"\n", " if (isinstance(item, list) and item[0] == \"\") or (isinstance(item, str) and item.startswith('')):\n", " return \"text\"\n", " if isinstance(item, list):\n", " return \"list\"\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "async def readBinaryRecord():\n", " \n", " print(\"begin of readBinaryRecord().\")\n", " LE = []\n", " Type = await reader.read(1)\n", " \n", " print(Type)\n", " print(\"Type: \" + str(Type))\n", " Type_dict = {\"BIN_LONGLIST\": 0,\n", " \"BIN_INTEGER\": 1,\n", " \"BIN_REAL\": 2,\n", " \"BIN_BOOLEAN\": 3,\n", " \"BIN_LONGSTRING\": 4,\n", " \"BIN_LONGSYMBOL\": 5,\n", " \"BIN_LONGTEXT\": 6,\n", " \"BIN_LIST\": 10,\n", " \"BIN_SHORTLIST\": 11,\n", " \"BIN_SHORTINT\": 12,\n", " \"BIN_BYTE\":13,\n", " \"BIN_STRING\": 14,\n", " \"BIN_SHORTSTRING\": 15,\n", " \"BIN_SYMBOL\": 16,\n", " \"BIN_SHORTSYMBOL\": 17,\n", " \"BIN_TEXT\": 18,\n", " \"BIN_SHORTTEXT\": 19,\n", " \"BIN_DOUBLE\": 20}\n", " typeCode = None\n", " for typ, val in Type_dict.items():\n", " if typ == Type:\n", " typeCode = val\n", " break\n", " if typeCode == 0: #LongList\n", " LE = []\n", " barray = bytearray(4) ####readInt()\n", " barray = await reader.readexactly(4)\n", " assert (int.from_bytes(barray, order = 'big', signed = True)) >= 0, \"Invalid Integer-Byte read from Stream!\"\n", " r = struct.unpack('>I',barray)\n", " ####\n", " if r == 0:\n", " return LE\n", " LE = await readBinaryRecord()\n", " Last = LE\n", " Next = []\n", " for i in range(r - 1):\n", " Next = await readBinaryRecord()\n", " if not Next:\n", " return None\n", " Last = Last.append(Next[0])\n", " return LE\n", " \n", " elif typeCode == 1:\n", " ####readInt()\n", " barray = bytearray(4)\n", " barray = await reader.readexactly(4)\n", " assert (int.from_bytes(barray, order = 'big', signed = True)) >= 0, \"Invalid Integer-Byte read from Stream!\"\n", " r = struct.unpack('>I',barray)\n", " ####\n", " LE[0] = r\n", " return LE \n", " \n", " elif typeCode == 2:\n", " ####readReal()\n", " barray = bytearray(4)\n", " barray = await reader.readexactly(4)\n", " assert (int.from_bytes(barray, order = 'big', signed = True)) >= 0, \"Invalid Real-Byte read from Stream!\"\n", " r = struct.unpack('>f',barray)\n", " ####\n", " LE[0] = r\n", " return LE \n", " \n", " elif typeCode == 3:\n", " ####readBool()\n", " barray = bytearray(1)\n", " barray = await reader.readexactly(1)\n", " assert (int.from_bytes(barray, order = 'big', signed = True)) >= 0, \"Invalid Boolean-Byte read from Stream!\"\n", " r = struct.unpack('?',barray)\n", " ####\n", " LE[0] = r\n", " return LE\n", " \n", " \n", " elif typeCode == 4:\n", " ####readLongString()\n", " barray = bytearray(4) #readInt()\n", " barray = await reader.readexactly(4)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid Int-Byte read from Stream!\"\n", " len = unpack('>I',barray)\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid LongSymbol-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " LE[0] = barray.decode()\n", " return LE\n", " \n", " elif typeCode == 5:\n", " ####readLongSymbol()\n", " barray = bytearray(4) #readInt()\n", " barray = await reader.readexactly(4)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid Int-Byte read from Stream!\"\n", " len = unpack('>I',barray)\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid LongSymbol-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " LE[0] = barray.decode()\n", " return LE\n", " \n", " elif typeCode == 6:\n", " ####readLongText()\n", " barray = bytearray(4) #readInt()\n", " barray = await reader.readexactly(4)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid Int-Byte read from Stream!\"\n", " len = unpack('>I',barray)\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid LongText-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " LE[0] = barray.decode()\n", " return LE\n", " \n", " elif typeCode == 10: #List\n", " LE = []\n", " ####readShort()\n", " barray = bytearray(2)\n", " barray = await reader.readexactly(2)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid ShortInt-Byte read from Stream!\"\n", " r = unpack('>H',barray)\n", " ####\n", " if r == 0:\n", " return LE\n", " LE = await readBinaryRecord()\n", " Last = LE\n", " Next = []\n", " for i in range(r - 1):\n", " Next = await readBinaryRecord()\n", " if not Next:\n", " return None\n", " Last = Last.append(Next[0])\n", " return LE\n", " \n", " elif typeCode == 11: #ShortList\n", " LE = []\n", " ####readByte()\n", " barray = bytearray(1)\n", " barray = await reader.readexactly(1)\n", " assert (int.from_bytes(barray, order = 'big', signed = True)) >= 0, \"Invalid Byte read from Stream!\"\n", " r = int.from_bytes(barray, order = 'big', signed = False)\n", " ####\n", " if r == 0:\n", " return LE\n", " LE = await readBinaryRecord()\n", " Last = LE\n", " Next = []\n", " for i in range(r - 1):\n", " Next = await readBinaryRecord()\n", " if not Next:\n", " return None\n", " Last = Last.append(Next[0])\n", " return LE\n", " \n", " elif typeCode == 12:\n", " ####readShort()\n", " barray = bytearray(2)\n", " barray = await reader.readexactly(2)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid ShortInt-Byte read from Stream!\"\n", " r = unpack('>H',barray)\n", " LE[0] = r\n", " return LE\n", " \n", " elif typeCode == 13:\n", " ####readByte()\n", " barray = bytearray(1)\n", " barray = await reader.readexactly(1)\n", " assert (int.from_bytes(barray, order = 'big', signed = True)) >= 0, \"Invalid Byte read from Stream!\"\n", " LE[0] = barray\n", " return LE\n", " \n", " elif typeCode == 14:\n", " ####readString()\n", " barray = bytearray(2)\n", " barray = await reader.readexactly(2)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid ShortInt-Byte read from Stream!\"\n", " len = unpack('>H',barray)\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid String-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " LE[0] = barray.decode()\n", " return LE\n", " \n", " elif typeCode == 15:\n", " ####readShortString()\n", " barray = bytearray(1)\n", " barray = await reader.readexactly(1)\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid Byte read from Stream!\"\n", " len = int.from_bytes(barray, order = 'big', signed = False)\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid ShortString-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " LE[0] = barray.decode()\n", " return LE\n", " \n", " elif typeCode == 16:\n", " ####readSymbol()\n", " barray = bytearray(2) #readShort()\n", " barray = await reader.readexactly(2)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid ShortInt-Byte read from Stream!\"\n", " len = unpack('>H',barray)\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid String-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " LE[0] = barray.decode()\n", " return LE\n", " \n", " elif typeCode == 17:\n", " ####readShortSymbol()\n", " barray = bytearray(1) #readByte()\n", " barray = await reader.readexactly(1)\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid Byte read from Stream!\"\n", " len = int.from_bytes(barray, order = 'big', signed = False)\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid ShortString-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " LE[0] = barray.decode()\n", " return LE\n", " \n", " elif typeCode == 18:\n", " ####readText()\n", " barray = bytearray(2) #readShort()\n", " barray = await reader.readexactly(2)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid ShortInt-Byte read from Stream!\"\n", " len = unpack('>H',barray)\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid Text-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " LE[0] = barray.decode()\n", " return LE\n", " \n", " elif typeCode == 19:\n", " ####readShortText()\n", " barray = bytearray(1) #readByte()\n", " barray = await reader.readexactly(1)\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid Byte read from Stream!\"\n", " len = int.from_bytes(barray, order = 'big', signed = False)\n", " barray = bytearray(len)\n", " barray = await reader.readexactly(len)\n", " for b in barray:\n", " assert (int.from_bytes(b, order = 'big', signed = True)) >= 0, \"Invalid ShortText-Byte read from Stream!\"\n", " ###checking Environment.ENCODING not considered yet\n", " LE[0] = barray.decode()\n", " return LE\n", " \n", " elif typeCode == 20:\n", " ####readDouble()\n", " barray = bytearray(8)\n", " barray = await reader.readexactly(8)\n", " assert (int.from_bytes(barray, order = 'big', signed = True)) >= 0, \"Invalid Double-Byte read from Stream!\"\n", " r = struct.unpack('>d',barray)\n", " ####\n", " LE[0] = r\n", " return LE\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "###################################################\n", "\n", "\n", "binLists = False\n", "async def Receive_List(finalToken):\n", " print(\"begin of Receive_List()\")\n", " answerList = []\n", " if not binLists:\n", " result = []\n", " count = 1\n", " print(\"binarylist = :\" str(binLists))\n", " while True: \n", " print(\"first line in Receive_List() read.\")\n", " line = await reader.readline()\n", " print(str(count) + \"th line read in Receive_List(): \" + line)\n", " line = line.decode().rstrip()\n", " count += 1\n", " if finalToken in line:\n", " break\n", " if line:\n", " result.append(line + \"\\n\")\n", " #result.append(\"\\n\")\n", " else:\n", " print(\"Network Error!\")\n", " return None\n", " \n", " \n", " #answerList = result\n", " print(\"NLparse: \\n\")\n", " result = '(' + ' '.join(result) + ')'\n", " print(result)\n", " answerList = NLparse(result, False)\n", " return answerList[0]\n", " else: #binLists = True --> readBinaryFrom(inputStream)\n", " #readString()\n", " print(\"binarylist = True\")\n", " line = await reader.read(3)\n", " signature = line.decode().rstrip()\n", " print(\"signature: \" + signature)\n", " \n", " #readShort()\n", " barray = bytearray(2)\n", " barray = await reader.read(2)\n", " major = unpack('>H',barray)\n", " major = major[0]\n", " print(\"major: \" + str(major))\n", " \n", " barray = await reader.read(2)\n", " minor = unpack('>H',barray)\n", " minor = minor[0]\n", " print(\"minor: \" + str(minor))\n", " \n", " \n", " if signature != \"bnl\":\n", " print(signature + \" is wrong Signatur!\")\n", " return None\n", " if (major != 1 or (minor != 0 and minor != 1 and minor != 2)):\n", " print(str(major) + \".\" + str(minor) + \" is wrong Version Number!\")\n", " return None\n", " print(\"readBinaryRecords() is called.\")\n", " answerList = await readBinaryRecord()\n", " return answerList\n", "\n", "\n", "####################################################\n", "\n", "\n", "async def Receive_Response():\n", " print(\"We r in Receive_Response() before reading.\")\n", " line = await reader.readline()\n", " line = line.decode().rstrip()\n", " print(\"first line in Receive_Response() read:\" + line)\n", " if not line:\n", " print(\"Network Error!\")\n", " return None\n", " count = 2\n", " while line == \"\":\n", " print(\"Receive_List() called.\")\n", " Message_List = await Receive_List(\"\")\n", " if not Message_List:\n", " print(\"Error- MessageList is empty!\")\n", " return None\n", " line = await reader.readline()\n", " line = line.decode().rstrip()\n", " print(str(count) + \"th line in Receive_Response() read:\" + line)\n", " count += 1\n", " if not line:\n", " print(\"Network Error!\")\n", " return None\n", " \n", " if line == \"\":\n", " line = await reader.readline()\n", " line = line.decode().rstrip()\n", " print(str(count) + \"th line in Receive_Response() after read:\" + line)\n", " count += 1\n", " if line:\n", " print(line)\n", " line = await reader.readline()\n", " line = line.decode().rstrip()\n", " print(str(count) + \"th line in Receive_Response() after read:\" + line)\n", " count += 1\n", " \n", " else:\n", " print(\"Error reading SecondoError!\")\n", " return None\n", " return None\n", " \n", " if line != \"\":\n", " print(\"Protocol Erroe!\")\n", " return None\n", " \n", " print(\"Receive_List() is called.\")\n", " answerList = await Receive_List(\"\")\n", " if not answerList:\n", " return None\n", " print(answerList)\n", " Error_Code = answerList[0]\n", " Error_Pos = answerList[1]\n", " Error_Message = answerList[2]\n", " Result_List = answerList[3]\n", " return Result_List" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "#NLresult = []\n", "\n", "binLists = False\n", "ftype = False\n", "\n", "#Command format: restore [database] database_name from \n", "async def restore_command(reader, writer, command):\n", " \n", " if reader and writer:\n", " if not command.startswith('(') :\n", " command = '(' + command.strip() + ')'\n", " \n", " CmdNL = NLparse(command, ftype)[0]\n", " print(CmdNL)\n", " length = len(CmdNL)\n", " if length != 4 and length != 5:\n", " print(\"restore_command(): command lenght wrong!\") \n", " return\n", " if length == 5: #restore database\n", " if CmdNL[0] != 'restore' or CmdNL[1] != 'database' or CmdNL[3] != 'from':\n", " print(\"restore_command: command wrong\") \n", " return\n", " name = CmdNL[2]\n", " filename = CmdNL[4]\n", " database = True\n", " if length == 4:\n", " if CmdNL[0] != 'restore' or CmdNL[2] != 'from':\n", " print(\"restore_command: command wrong\") \n", " return\n", " name = CmdNL[1]\n", " filename = CmdNL[3]\n", " database = False\n", " \n", " typ = AtomType(filename)\n", " if not typ in (\"string\", \"symbol\", \"text\"):\n", " print(\"restore_command: filename wrong\") \n", " return\n", " \n", " \n", " tag = \"DbRestore\" if database else \"ObjectRestore\"\n", " message = \"<\" + tag + \">\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", " \n", "\n", " message = name + \"\\n\"\n", " print(\"name:\")\n", " print(name)\n", " message = message.encode()\n", " writer.write(message)\n", " \n", " \n", " message = \"\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", " \n", " with open(filename, encoding=\"utf8\", errors='ignore') as f:\n", " \n", " #flen = os.path.getsize(f.name)\n", " file = f.read()\n", " flen = len(file)\n", " print(flen)\n", " message = str(flen) + \"\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", " await writer.drain()\n", " \n", " #contents of file\n", " message = file\n", " message = message.encode()\n", " writer.write(message)\n", " \n", " \n", " message = \"\\n\"\n", " message = message.encode()\n", " #print(message)\n", " writer.write(message)\n", " \n", " message = \"\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", " await writer.drain()\n", " #print(message)\n", " \n", " #### secondoresponse- ReceiveResponse\n", " print(\"calling Receive_Response()\")\n", " resp = await Receive_Response()\n", " print(resp)\n", " \n", " return resp\n", " \n", " else:\n", " print(\"reader and writer Sockets null\")\n", " return None\n", "\n", " \n", "\n", "async def perform_command(reader, writer, command):\n", " if command.lstrip().startswith('(restore') or command.lstrip().startswith('( restore') or command.lstrip().startswith('restore'):\n", " result = await restore_command(reader, writer, command)\n", " return result\n", "\n", "\n", "command = '(restore database berlintest from berlintest)' \n", "#rs = async_loop.run_until_complete(perform_command(reader, writer, command))\n", "rs = await perform_command(reader, writer, command)\n", "\n", "#cr = perform_command(command)\n", "#loop = asyncio.get_event_loop()\n", "#rs = loop.run_until_complete(cr)\n", "\n", "#print(rs)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#finding the index of an item in a list\n", "#index(...)\n", "#L.index(value, [start, [stop]]) -> integer -- return first index of value\n", "\n", "#\n", "#i = iter(lisst)\n", "#first = next(i)\n", "#rest = list(i)\n", "\n", "\n", "#recursive approach to itterate a nested list of unknown depth and structure\n", "#def traverse(o, tree_types=(list, tuple)):\n", "# if isinstance(o, tree_types):\n", "# for value in o:\n", "# for subvalue in traverse(value, tree_types):\n", "# yield subvalue\n", "# else:\n", "# yield o\n", "\n", "#a = number of inner lists\n", "#lst = [[] for _ in range(a)]\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ " message = \"\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", " \n", " cmdLevel = \"0\" + \"\\n\"\n", " message = cmdLevel.encode()\n", " writer.write(message)\n", " \n", " \n", " message = \"open databese berlintest\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", "\n", " \n", " \n", " \n", " message = \"\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", " await writer.drain()\n", "\n", " print(\"start reading.\")\n", " line = await reader.readline()\n", " line = line.decode().rstrip()\n", " print(line)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ " tag = \"DbRestore\"\n", " message = \"<\" + tag + \">\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", " \n", " name = \"berlintest\"\n", " message = name + \"\\n\"\n", " print(\"name:\")\n", " print(name)\n", " message = message.encode()\n", " writer.write(message)\n", " \n", " \n", " message = \"\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", " \n", " filename = \"berlintest\"\n", " with open(filename, encoding=\"utf8\", errors='ignore') as f:\n", " \n", " flen = os.path.getsize(f.name)\n", " print(flen)\n", " message = str(flen) + \"\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", " await writer.drain()\n", " \n", " #contents of file\n", " message = f.read()\n", " message = message.encode()\n", " writer.write(message)\n", " \n", " \n", " message = \"\\n\"\n", " message = message.encode()\n", " #print(message)\n", " writer.write(message)\n", " \n", " message = \"\\n\"\n", " message = message.encode()\n", " writer.write(message)\n", " await writer.drain()\n", " \n", " print(\"Start reading\")\n", " #await asyncio.wait_for(reader.readline(),timeout=2)\n", " line = await reader.readline()\n", " if line:\n", " line = line.decode()\n", " print(line)\n", " else:\n", " print(\"empty line!\")\n", " \n", " print(\"End reading\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from struct import *\n", "unpack('>H',b'\\x00\\x13')\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "int('\\x13',16)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.4" } }, "nbformat": 4, "nbformat_minor": 2 }