From d6aa55704a77b32a5bcd9d15cdf99fd2c719ab13 Mon Sep 17 00:00:00 2001
From: Kaabp <contact@kaabp.de>
Date: Mon, 28 Feb 2022 22:55:07 +0100
Subject: [PATCH] First Commit

---
 .idea/.gitignore                         |   8 ++++++
 requirements.txt                         | Bin 0 -> 116 bytes
 sample/file_handling/add_to_csv.py       |  23 +++++++++++++++++
 sample/file_handling/create_basic_csv.py |   8 ++++++
 sample/file_handling/merge_csvs.py       |   0
 sample/file_handling/read_csv.py         |  13 ++++++++++
 sample/main.py                           |  30 +++++++++++++++++++++++
 sample/query/query_part_names.py         |  19 ++++++++++++++
 sample/query/query_part_quantity.py      |  27 ++++++++++++++++++++
 sample/query/query_parts.py              |  21 ++++++++++++++++
 sample/query/query_set.py                |  13 ++++++++++
 sample/util/util.py                      |  15 ++++++++++++
 tests/test_part_numbers.py               |  10 ++++++++
 13 files changed, 187 insertions(+)
 create mode 100644 .idea/.gitignore
 create mode 100644 requirements.txt
 create mode 100644 sample/file_handling/add_to_csv.py
 create mode 100644 sample/file_handling/create_basic_csv.py
 create mode 100644 sample/file_handling/merge_csvs.py
 create mode 100644 sample/file_handling/read_csv.py
 create mode 100644 sample/main.py
 create mode 100644 sample/query/query_part_names.py
 create mode 100644 sample/query/query_part_quantity.py
 create mode 100644 sample/query/query_parts.py
 create mode 100644 sample/query/query_set.py
 create mode 100644 sample/util/util.py
 create mode 100644 tests/test_part_numbers.py

diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 0000000..1c2fda5
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..57f05a3749e0a59b91dea68e5f6978d9cf0518b9
GIT binary patch
literal 116
zcmX|(!3sb?7=*vG@+#V8uO5c1Qetay^yrwM(lj%N`I`IciFE92OpNN8%&JZVS}9QB
haY{!?@vm8Y<dBW3wKp$mj5+CbVNhmH5nk)}^#B)%6Ab_W

literal 0
HcmV?d00001

diff --git a/sample/file_handling/add_to_csv.py b/sample/file_handling/add_to_csv.py
new file mode 100644
index 0000000..12f3b5f
--- /dev/null
+++ b/sample/file_handling/add_to_csv.py
@@ -0,0 +1,23 @@
+import csv
+from itertools import zip_longest
+from file_handling import read_csv
+import pandas as pd
+
+
+def add_list_to_empty_csv(file, set_list):
+    rows = zip_longest(*set_list, fillvalue="")
+    with open(file, "w", encoding="ISO-8859-1", newline='') as file:
+        writer = csv.writer(file)
+        writer.writerows(rows)
+    file.close()
+
+
+def add_df_to_empty_csv(file, setlist_df):
+    setlist_df.to_csv(file, index=False)
+
+
+def append_df_csv(file, setlist_df):
+    csv_list = read_csv.return_csv_as_df(file)
+    setlist_df["Quantity"] = setlist_df["Quantity"].astype(int)
+    df = pd.concat([csv_list, setlist_df]).groupby(["Part number", "Name"], as_index=False)["Quantity"].sum()
+    df.to_csv(file, index=False)
diff --git a/sample/file_handling/create_basic_csv.py b/sample/file_handling/create_basic_csv.py
new file mode 100644
index 0000000..19de6a5
--- /dev/null
+++ b/sample/file_handling/create_basic_csv.py
@@ -0,0 +1,8 @@
+import csv
+
+
+def basic_csv(file):
+    with open(file, "w", encoding="ISO-8859-1", newline='') as file:
+        writer = csv.writer(file)
+        writer.writerow(("Part number", "Name", "Quantity"))
+    file.close()
diff --git a/sample/file_handling/merge_csvs.py b/sample/file_handling/merge_csvs.py
new file mode 100644
index 0000000..e69de29
diff --git a/sample/file_handling/read_csv.py b/sample/file_handling/read_csv.py
new file mode 100644
index 0000000..ae7c3b7
--- /dev/null
+++ b/sample/file_handling/read_csv.py
@@ -0,0 +1,13 @@
+import csv
+import pandas as pd
+
+
+def return_csv_as_df(csv_file):
+    return pd.read_csv(csv_file)
+
+
+def print_csv(csv_file):
+    with open(csv_file, "r") as file:
+        reader = csv.reader(file)
+        for row in reader:
+            print(row)
\ No newline at end of file
diff --git a/sample/main.py b/sample/main.py
new file mode 100644
index 0000000..156b293
--- /dev/null
+++ b/sample/main.py
@@ -0,0 +1,30 @@
+import query.query_set
+from query.query_set import return_as_df as df
+from query.query_set import return_as_list
+from util.util import get_items
+from query.query_part_quantity import get_part_quantity, __find_parts_quantity
+from file_handling import read_csv, add_to_csv
+
+url00 = "https://www.bricklink.com/CatalogItemInv.asp?S=75979-1"
+url01 = "https://www.bricklink.com/CatalogItemInv.asp?S=76387-1"
+url02 = "https://www.bricklink.com/CatalogItemInv.asp?S=75308-1"
+url03 = "https://www.bricklink.com/CatalogItemInv.asp?S=75192-1" # <-- 7000+ pieces
+
+#lists = get_items(url00)
+#parts = __find_parts_quantity(lists, 184) #185
+#print(parts)
+
+
+lists = get_part_quantity(url00)
+
+with open("test.txt", "w") as file:
+    for items in lists:
+        file.writelines([items])
+file.close()
+
+# set_list = return_as_list(url00)
+# add_to_csv.append_df_csv("test01.csv", df)
+
+# add_to_csv.add_list_to_empty_csv("test.csv", set_list)
+
+# print(df.to_string(index=False))
diff --git a/sample/query/query_part_names.py b/sample/query/query_part_names.py
new file mode 100644
index 0000000..2fce28e
--- /dev/null
+++ b/sample/query/query_part_names.py
@@ -0,0 +1,19 @@
+from sample.util.util import findnth, get_items
+
+
+def __find_part_name(parts_list, idx):
+    return parts_list[idx][(parts_list[idx].find('<td><b>') + 7):findnth(parts_list[idx], '</b>', 1)]
+
+
+def get_part_names(url):
+    items = get_items(url)
+    part_numbers = []
+
+    for i in range(len(items)):
+        current_part = __find_part_name(items, i)
+        if "<" in current_part:
+            continue
+        elif i >= 4:
+            part_numbers.append(current_part)
+
+    return part_numbers
\ No newline at end of file
diff --git a/sample/query/query_part_quantity.py b/sample/query/query_part_quantity.py
new file mode 100644
index 0000000..c580378
--- /dev/null
+++ b/sample/query/query_part_quantity.py
@@ -0,0 +1,27 @@
+from sample.util.util import findnth, get_items
+
+
+def __find_parts_quantity(parts_list, idx):
+    return parts_list[idx][(parts_list[idx].find('<td align="RIGHT">') + 19):findnth(parts_list[idx], '</td>', 1) - 1]
+    # current_part = parts_list[idx][(parts_list[idx].find('<td align="RIGHT">') + 19):
+    #                                findnth(parts_list[idx], '</td>', 1) - 1]
+    # if current_part == '><td align="CENTER" colspan="5"><font color="#FFFFFF" face="Tahoma,Arial" size="2"><b>Counterparts:</b></font></td></t':
+    #     return "NaN"
+    # else:
+    #     return current_part
+
+
+def get_part_quantity(url):
+    items = get_items(url)
+    part_numbers = []
+
+    for i in range(len(items)):
+        current_part = __find_parts_quantity(items, i)
+        # if current_part == "NaN":
+        #     break
+        # elif len(current_part) < 30:
+        #     part_numbers.append(current_part)
+        if len(current_part) < 30:
+            part_numbers.append(current_part)
+
+    return part_numbers
diff --git a/sample/query/query_parts.py b/sample/query/query_parts.py
new file mode 100644
index 0000000..e8ff3b3
--- /dev/null
+++ b/sample/query/query_parts.py
@@ -0,0 +1,21 @@
+from sample.util.util import findnth, get_items
+from bs4 import BeautifulSoup
+from urllib.request import Request, urlopen
+
+
+def __find_part_number(parts_list, idx):
+    return parts_list[idx][(parts_list[idx].find("title=") + 16):findnth(parts_list[idx], "  Name:", 1)]
+
+
+def get_part_number(url):
+    items = get_items(url)
+    part_numbers = []
+
+    for i in range(len(items)):
+        current_part = __find_part_number(items, i)
+        if current_part.startswith("o:"):
+            current_part = current_part[3:30]
+        if len(current_part) < 30:
+            part_numbers.append(current_part)
+
+    return part_numbers
diff --git a/sample/query/query_set.py b/sample/query/query_set.py
new file mode 100644
index 0000000..c3ec68e
--- /dev/null
+++ b/sample/query/query_set.py
@@ -0,0 +1,13 @@
+from query import query_parts, query_part_names, query_part_quantity
+import pandas as pd
+
+
+def return_as_list(url):
+    return [query_parts.get_part_number(url), query_part_names.get_part_names(url),
+            query_part_quantity.get_part_quantity(url)]
+
+
+def return_as_df(url):
+    lst = return_as_list(url)
+    df = pd.DataFrame({"Part number": lst[0], "Name": lst[1], "Quantity": lst[2]}, index="false")
+    return df
diff --git a/sample/util/util.py b/sample/util/util.py
new file mode 100644
index 0000000..5aaf270
--- /dev/null
+++ b/sample/util/util.py
@@ -0,0 +1,15 @@
+from urllib.request import urlopen, Request
+from bs4 import BeautifulSoup
+
+
+def findnth(list_, keyword, n):
+    parts = list_.split(keyword, n + 1)
+    if len(parts) <= n + 1:
+        return -1
+    return len(list_) - len(parts[-1]) - len(keyword)
+
+
+def get_items(url):
+    src = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
+    webpage = urlopen(src).read()
+    return str(BeautifulSoup(webpage, "html.parser").find("table", class_="ta")).split("<tr")
diff --git a/tests/test_part_numbers.py b/tests/test_part_numbers.py
new file mode 100644
index 0000000..b9828a2
--- /dev/null
+++ b/tests/test_part_numbers.py
@@ -0,0 +1,10 @@
+import unittest
+from sample.query import query_parts
+
+
+class TestPartNumber(unittest.TestCase):
+
+    def test_part_numbers(self, url):
+        test = query_parts.get_part_number("https://www.bricklink.com/CatalogItemInv.asp?S=75308-1")
+        self.assertEqual(test[150], "3034")
+