First Commit

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

sample/file_handling/add_to_csv.py

@@ -0,0 +1,23 @@
+import csv
+from itertools import zip_longest
+from file_handling import read_csv
+import pandas as pd
+
+
+def add_list_to_empty_csv(file, set_list):
+    rows = zip_longest(*set_list, fillvalue="")
+    with open(file, "w", encoding="ISO-8859-1", newline='') as file:
+        writer = csv.writer(file)
+        writer.writerows(rows)
+    file.close()
+
+
+def add_df_to_empty_csv(file, setlist_df):
+    setlist_df.to_csv(file, index=False)
+
+
+def append_df_csv(file, setlist_df):
+    csv_list = read_csv.return_csv_as_df(file)
+    setlist_df["Quantity"] = setlist_df["Quantity"].astype(int)
+    df = pd.concat([csv_list, setlist_df]).groupby(["Part number", "Name"], as_index=False)["Quantity"].sum()
+    df.to_csv(file, index=False)

sample/file_handling/create_basic_csv.py

@@ -0,0 +1,8 @@
+import csv
+
+
+def basic_csv(file):
+    with open(file, "w", encoding="ISO-8859-1", newline='') as file:
+        writer = csv.writer(file)
+        writer.writerow(("Part number", "Name", "Quantity"))
+    file.close()

sample/file_handling/read_csv.py

@@ -0,0 +1,13 @@
+import csv
+import pandas as pd
+
+
+def return_csv_as_df(csv_file):
+    return pd.read_csv(csv_file)
+
+
+def print_csv(csv_file):
+    with open(csv_file, "r") as file:
+        reader = csv.reader(file)
+        for row in reader:
+            print(row)

sample/main.py

@@ -0,0 +1,30 @@
+import query.query_set
+from query.query_set import return_as_df as df
+from query.query_set import return_as_list
+from util.util import get_items
+from query.query_part_quantity import get_part_quantity, __find_parts_quantity
+from file_handling import read_csv, add_to_csv
+
+url00 = "https://www.bricklink.com/CatalogItemInv.asp?S=75979-1"
+url01 = "https://www.bricklink.com/CatalogItemInv.asp?S=76387-1"
+url02 = "https://www.bricklink.com/CatalogItemInv.asp?S=75308-1"
+url03 = "https://www.bricklink.com/CatalogItemInv.asp?S=75192-1" # <-- 7000+ pieces
+
+#lists = get_items(url00)
+#parts = __find_parts_quantity(lists, 184) #185
+#print(parts)
+
+
+lists = get_part_quantity(url00)
+
+with open("test.txt", "w") as file:
+    for items in lists:
+        file.writelines([items])
+file.close()
+
+# set_list = return_as_list(url00)
+# add_to_csv.append_df_csv("test01.csv", df)
+
+# add_to_csv.add_list_to_empty_csv("test.csv", set_list)
+
+# print(df.to_string(index=False))

sample/query/query_part_names.py

@@ -0,0 +1,19 @@
+from sample.util.util import findnth, get_items
+
+
+def __find_part_name(parts_list, idx):
+    return parts_list[idx][(parts_list[idx].find('<td><b>') + 7):findnth(parts_list[idx], '</b>', 1)]
+
+
+def get_part_names(url):
+    items = get_items(url)
+    part_numbers = []
+
+    for i in range(len(items)):
+        current_part = __find_part_name(items, i)
+        if "<" in current_part:
+            continue
+        elif i >= 4:
+            part_numbers.append(current_part)
+
+    return part_numbers

sample/query/query_part_quantity.py

@@ -0,0 +1,27 @@
+from sample.util.util import findnth, get_items
+
+
+def __find_parts_quantity(parts_list, idx):
+    return parts_list[idx][(parts_list[idx].find('<td align="RIGHT">') + 19):findnth(parts_list[idx], '</td>', 1) - 1]
+    # current_part = parts_list[idx][(parts_list[idx].find('<td align="RIGHT">') + 19):
+    #                                findnth(parts_list[idx], '</td>', 1) - 1]
+    # if current_part == '><td align="CENTER" colspan="5"><font color="#FFFFFF" face="Tahoma,Arial" size="2"><b>Counterparts:</b></font></td></t':
+    #     return "NaN"
+    # else:
+    #     return current_part
+
+
+def get_part_quantity(url):
+    items = get_items(url)
+    part_numbers = []
+
+    for i in range(len(items)):
+        current_part = __find_parts_quantity(items, i)
+        # if current_part == "NaN":
+        #     break
+        # elif len(current_part) < 30:
+        #     part_numbers.append(current_part)
+        if len(current_part) < 30:
+            part_numbers.append(current_part)
+
+    return part_numbers

sample/query/query_parts.py

@@ -0,0 +1,21 @@
+from sample.util.util import findnth, get_items
+from bs4 import BeautifulSoup
+from urllib.request import Request, urlopen
+
+
+def __find_part_number(parts_list, idx):
+    return parts_list[idx][(parts_list[idx].find("title=") + 16):findnth(parts_list[idx], "  Name:", 1)]
+
+
+def get_part_number(url):
+    items = get_items(url)
+    part_numbers = []
+
+    for i in range(len(items)):
+        current_part = __find_part_number(items, i)
+        if current_part.startswith("o:"):
+            current_part = current_part[3:30]
+        if len(current_part) < 30:
+            part_numbers.append(current_part)
+
+    return part_numbers

sample/query/query_set.py

@@ -0,0 +1,13 @@
+from query import query_parts, query_part_names, query_part_quantity
+import pandas as pd
+
+
+def return_as_list(url):
+    return [query_parts.get_part_number(url), query_part_names.get_part_names(url),
+            query_part_quantity.get_part_quantity(url)]
+
+
+def return_as_df(url):
+    lst = return_as_list(url)
+    df = pd.DataFrame({"Part number": lst[0], "Name": lst[1], "Quantity": lst[2]}, index="false")
+    return df

sample/util/util.py

@@ -0,0 +1,15 @@
+from urllib.request import urlopen, Request
+from bs4 import BeautifulSoup
+
+
+def findnth(list_, keyword, n):
+    parts = list_.split(keyword, n + 1)
+    if len(parts) <= n + 1:
+        return -1
+    return len(list_) - len(parts[-1]) - len(keyword)
+
+
+def get_items(url):
+    src = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
+    webpage = urlopen(src).read()
+    return str(BeautifulSoup(webpage, "html.parser").find("table", class_="ta")).split("<tr")

tests/test_part_numbers.py

@@ -0,0 +1,10 @@
+import unittest
+from sample.query import query_parts
+
+
+class TestPartNumber(unittest.TestCase):
+
+    def test_part_numbers(self, url):
+        test = query_parts.get_part_number("https://www.bricklink.com/CatalogItemInv.asp?S=75308-1")
+        self.assertEqual(test[150], "3034")
+