Add GitHub stats for custom nodes (#533)

* Add GitHub stats fetching feature
- Added PyGithub package to requirements.txt for GitHub API interaction
- Updated .gitignore to ignore github-stats-cache.json
- Produced github-stats.json for storing GitHub stats
- Modified scanner.py to include the GitHub stats fetching process

* Add sorting for 'GitHub Stars' and 'Last Update' columns

- Fetch 'GitHub Stars' and 'Last Update' data when getting the custom node list.
- Display 'GitHub Stars' and 'Last Update' information in the UI.
- Implement sorting functionality for these two columns, allowing users to sort both in descending and ascending order.

* fix: scanner - prevent stuck when exceed rate limit

---------

Co-authored-by: Dr.Lt.Data <dr.lt.data@gmail.com>

.gitignore

@@ -11,3 +11,4 @@ startup-scripts/**
 matrix_auth
 channels.list
 comfyworkflows_sharekey
+github-stats-cache.json

__init__.py

@@ -621,6 +621,20 @@ async def get_data(uri, silent=False):
     json_obj = json.loads(json_text)
     return json_obj
 
+async def populate_github_stats(json_obj, filename, silent=False):
+    uri = os.path.join(comfyui_manager_path, filename)
+    with open(uri, "r", encoding='utf-8') as f:
+        github_stats = json.load(f)
+    if 'custom_nodes' in json_obj:
+        for i, node in enumerate(json_obj['custom_nodes']):
+            url = node['reference']
+            if url in github_stats:
+                json_obj['custom_nodes'][i]['stars'] = github_stats[url]['stars']
+                json_obj['custom_nodes'][i]['last_update'] = github_stats[url]['last_update']
+            else:
+                json_obj['custom_nodes'][i]['stars'] = -1
+                json_obj['custom_nodes'][i]['last_update'] = -1
+        return json_obj
 
 def setup_js():
     import nodes
@@ -1005,6 +1019,7 @@ async def fetch_customnode_list(request):
         channel = get_config()['channel_url']
 
     json_obj = await get_data_by_mode(request.rel_url.query["mode"], 'custom-node-list.json')
+    json_obj = await populate_github_stats(json_obj, "github-stats.json")
 
     def is_ignored_notice(code):
         global version

js/custom-nodes-downloader.js

@@ -109,6 +109,9 @@ export class CustomNodesInstaller extends ComfyDialog {
 		this.manager_dialog = manager_dialog;
 		this.search_keyword = '';
 		this.element = $el("div.comfy-modal", { parent: document.body }, []);
+
+		this.currentSortProperty = ''; // The property currently being sorted
+		this.currentSortAscending = true; // The direction of the current sort		
 	}
 
 	startInstall(target) {
@@ -367,16 +370,79 @@ export class CustomNodesInstaller extends ComfyDialog {
 		}
 	}
 
+	sortData(property, ascending = true) {
+		this.data.sort((a, b) => {
+			// Check if either value is -1 and handle accordingly
+			if (a[property] === -1) return 1; // Always put a at the end if its value is -1
+			if (b[property] === -1) return -1; // Always put b at the end if its value is -1
+			// And be careful here, (-1<'2024-01-01') and (-1>'2024-01-01') are both false! So I handle -1 seperately.
+			if (a[property] < b[property]) return ascending ? -1 : 1;
+			if (a[property] > b[property]) return ascending ? 1 : -1;
+			return 0;
+		});
+	}
+
+	resetHeaderStyles() {
+		const headers = ['th_stars', 'th_last_update']; // Add the IDs of all your sortable headers here
+		headers.forEach(headerId => {
+			const header = this.element.querySelector(`#${headerId}`);
+			if (header) {
+				header.style.backgroundColor = ''; // Reset to default background color
+				// Add other style resets if necessary
+			}
+		});
+	}
+	
+	toggleSort(property) {
+		// If currently sorted by this property, toggle the direction; else, sort ascending
+		if (this.currentSortProperty === property) {
+			this.currentSortAscending = !this.currentSortAscending;
+		} else {
+			this.currentSortAscending = false;
+		}
+		this.currentSortProperty = property;
+	
+		this.resetHeaderStyles(); // Reset styles of all sortable headers
+
+		// Determine the ID of the header based on the property
+		let headerId = '';
+		if (property === 'stars') {
+			headerId = 'th_stars';
+		} else if (property === 'last_update') {
+			headerId = 'th_last_update';
+		}
+
+		// If we have a valid headerId, change its style to indicate it's the active sort column
+		if (headerId) {
+			const activeHeader = this.element.querySelector(`#${headerId}`);
+			if (activeHeader) {
+				activeHeader.style.backgroundColor = '#222';
+				// Slightly brighter. Add other style changes if necessary.
+			}
+		}
+		
+		// Call sortData with the current property and direction
+		this.sortData(property, this.currentSortAscending);
+	
+		// Refresh the grid to display sorted data
+		this.createGrid();
+	}
+
 	async createGrid() {
-		var grid = document.createElement('table');
+		// Remove existing table if present
+		var grid = this.element.querySelector('#custom-nodes-grid');
+		var panel;
+		let self = this;
+		if (grid) {
+			grid.querySelector('tbody').remove();
+			panel = grid.parentNode;
+		} else {
+			grid = document.createElement('table');
 			grid.setAttribute('id', 'custom-nodes-grid');
 
 			this.grid_rows = {};
 
-		let self = this;
-
 			var thead = document.createElement('thead');
-		var tbody = document.createElement('tbody');
 
 			var headerRow = document.createElement('tr');
 			thead.style.position = "sticky";
@@ -404,10 +470,22 @@ export class CustomNodesInstaller extends ComfyDialog {
 			var header4 = document.createElement('th');
 			header4.innerHTML = 'Description';
 			header4.style.width = "60%";
-//		header4.classList.add('expandable-column');
+	//		header4.classList.add('expandable-column');
 			var header5 = document.createElement('th');
-		header5.innerHTML = 'Install';
+			header5.innerHTML = 'GitHub Stars';
 			header5.style.width = "130px";
+			header5.setAttribute('id', 'th_stars');
+			header5.style.cursor = 'pointer';
+			header5.onclick = () => this.toggleSort('stars');
+			var header6 = document.createElement('th');
+			header6.innerHTML = 'Last Update';
+			header6.style.width = "130px";
+			header6.setAttribute('id', 'th_last_update');
+			header6.style.cursor = 'pointer';
+			header6.onclick = () => this.toggleSort('last_update');
+			var header7 = document.createElement('th');
+			header7.innerHTML = 'Install';
+			header7.style.width = "130px";
 
 			header0.style.position = "sticky";
 			header0.style.top = "0px";
@@ -421,6 +499,10 @@ export class CustomNodesInstaller extends ComfyDialog {
 			header4.style.top = "0px";
 			header5.style.position = "sticky";
 			header5.style.top = "0px";
+			header6.style.position = "sticky";
+			header6.style.top = "0px";
+			header7.style.position = "sticky";
+			header7.style.top = "0px";
 
 			thead.appendChild(headerRow);
 			headerRow.appendChild(header0);
@@ -429,6 +511,8 @@ export class CustomNodesInstaller extends ComfyDialog {
 			headerRow.appendChild(header3);
 			headerRow.appendChild(header4);
 			headerRow.appendChild(header5);
+			headerRow.appendChild(header6);
+			headerRow.appendChild(header7);
 
 			headerRow.style.backgroundColor = "Black";
 			headerRow.style.color = "White";
@@ -437,6 +521,13 @@ export class CustomNodesInstaller extends ComfyDialog {
 			headerRow.style.padding = "0";
 
 			grid.appendChild(thead);
+
+			panel = document.createElement('div');
+			panel.style.width = "100%";
+			panel.appendChild(grid);
+			this.element.appendChild(panel);
+		}
+		var tbody = document.createElement('tbody');
 		grid.appendChild(tbody);
 
 		if(this.data)
@@ -499,8 +590,27 @@ export class CustomNodesInstaller extends ComfyDialog {
 				}
 
 				var data5 = document.createElement('td');
+				data5.style.maxWidth = "100px";
+				data5.className = "cm-node-stars"
+				data5.textContent = `${data.stars}`;
+				data5.style.whiteSpace = "nowrap";
+				data5.style.overflow = "hidden";
+				data5.style.textOverflow = "ellipsis";
 				data5.style.textAlign = "center";
 
+				var lastUpdateDate = new Date();
+				var data6 = document.createElement('td');
+				data6.style.maxWidth = "100px";
+				data6.className = "cm-node-last-update"
+				data6.textContent = `${data.last_update}`.split(' ')[0];
+				data6.style.whiteSpace = "nowrap";
+				data6.style.overflow = "hidden";
+				data6.style.textOverflow = "ellipsis";
+				data6.style.textAlign = "center";
+
+				var data7 = document.createElement('td');
+				data7.style.textAlign = "center";
+
 				var installBtn = document.createElement('button');
 				installBtn.className = "cm-btn-install";
 				var installBtn2 = null;
@@ -587,7 +697,7 @@ export class CustomNodesInstaller extends ComfyDialog {
 						install_checked_custom_node(self.grid_rows, j, CustomNodesInstaller.instance, 'update');
 					});
 
-					data5.appendChild(installBtn2);
+					data7.appendChild(installBtn2);
 				}
 
 				if(installBtn3 != null) {
@@ -596,7 +706,7 @@ export class CustomNodesInstaller extends ComfyDialog {
 						install_checked_custom_node(self.grid_rows, j, CustomNodesInstaller.instance, 'toggle_active');
 					});
 
-					data5.appendChild(installBtn3);
+					data7.appendChild(installBtn3);
 				}
 
 				if(installBtn4 != null) {
@@ -605,7 +715,7 @@ export class CustomNodesInstaller extends ComfyDialog {
 						install_checked_custom_node(self.grid_rows, j, CustomNodesInstaller.instance, 'fix');
 					});
 
-					data5.appendChild(installBtn4);
+					data7.appendChild(installBtn4);
 				}
 
 				installBtn.style.width = "120px";
@@ -621,7 +731,7 @@ export class CustomNodesInstaller extends ComfyDialog {
 				});
 
 				if(!data.author.startsWith('#NOTICE')){
-					data5.appendChild(installBtn);
+					data7.appendChild(installBtn);
 				}
 
 				if(data.installed == 'Fail' || data.author.startsWith('#NOTICE'))
@@ -637,6 +747,8 @@ export class CustomNodesInstaller extends ComfyDialog {
 				dataRow.appendChild(data3);
 				dataRow.appendChild(data4);
 				dataRow.appendChild(data5);
+				dataRow.appendChild(data6);
+				dataRow.appendChild(data7);
 				tbody.appendChild(dataRow);
 
 				let buttons = [];
@@ -653,10 +765,6 @@ export class CustomNodesInstaller extends ComfyDialog {
 				this.grid_rows[i] = {data:data, buttons:buttons, checkbox:checkbox, control:dataRow};
 			}
 
-		const panel = document.createElement('div');
-		panel.style.width = "100%";
-		panel.appendChild(grid);
-
 		function handleResize() {
 		  const parentHeight = self.element.clientHeight;
 		  const gridHeight = parentHeight - 200;
@@ -672,7 +780,6 @@ export class CustomNodesInstaller extends ComfyDialog {
 		grid.style.overflowY = "scroll";
 		this.element.style.height = "85%";
 		this.element.style.width = "80%";
-		this.element.appendChild(panel);
 
 		handleResize();
 	}

requirements.txt

@@ -1,4 +1,5 @@
 GitPython
+PyGithub
 matrix-client==0.4.0
 transformers
 huggingface-hub>0.20

scanner.py

@@ -5,11 +5,16 @@ import json
 from git import Repo
 from torchvision.datasets.utils import download_url
 import concurrent
+import datetime
 
 builtin_nodes = set()
 
 import sys
 
+from urllib.parse import urlparse
+from github import Github
+
+g = Github(os.environ.get('GITHUB_TOKEN'))
 
 # prepare temp dir
 if len(sys.argv) > 1:
@@ -213,9 +218,6 @@ def update_custom_nodes():
     git_url_titles_preemptions = get_git_urls_from_json('custom-node-list.json')
 
     def process_git_url_title(url, title, preemptions, node_pattern):
-        if 'Jovimetrix' in title:
-            pass
-
         name = os.path.basename(url)
         if name.endswith(".git"):
             name = name[:-4]
@@ -224,7 +226,51 @@ def update_custom_nodes():
         if not skip_update:
             clone_or_pull_git_repository(url)
 
-    with concurrent.futures.ThreadPoolExecutor(10) as executor:
+    def process_git_stats(git_url_titles_preemptions):
+        GITHUB_STATS_CACHE_FILENAME = 'github-stats-cache.json'
+        GITHUB_STATS_FILENAME = 'github-stats.json'
+
+        github_stats = {}
+        try:
+            with open(GITHUB_STATS_CACHE_FILENAME, 'r', encoding='utf-8') as file:
+                github_stats = json.load(file)
+        except FileNotFoundError:
+            pass
+
+        if g.rate_limiting_resettime-datetime.datetime.now().timestamp() <= 0:
+            for url, title, preemptions, node_pattern in git_url_titles_preemptions:
+                if url not in github_stats:
+                    # Parsing the URL
+                    parsed_url = urlparse(url)
+                    domain = parsed_url.netloc
+                    path = parsed_url.path
+                    path_parts = path.strip("/").split("/")
+                    if len(path_parts) >= 2 and domain == "github.com":
+                        owner_repo = "/".join(path_parts[-2:])
+                        repo = g.get_repo(owner_repo)
+
+                        last_update = repo.pushed_at.strftime("%Y-%m-%d %H:%M:%S") if repo.pushed_at else 'N/A'
+                        github_stats[url] = {
+                            "stars": repo.stargazers_count,
+                            "last_update": last_update,
+                        }
+                        with open(GITHUB_STATS_CACHE_FILENAME, 'w', encoding='utf-8') as file:
+                            json.dump(github_stats, file, ensure_ascii=False, indent=4)
+                        # print(f"Title: {title}, Stars: {repo.stargazers_count}, Last Update: {last_update}")
+                    else:
+                        print(f"Invalid URL format for GitHub repository: {url}")
+
+        with open(GITHUB_STATS_FILENAME, 'w', encoding='utf-8') as file:
+            json.dump(github_stats, file, ensure_ascii=False, indent=4)
+
+        print(f"Successfully written to {GITHUB_STATS_FILENAME}, removing {GITHUB_STATS_CACHE_FILENAME}.")
+        try:
+            os.remove(GITHUB_STATS_CACHE_FILENAME) # This cache file is just for avoiding failure of GitHub API fetch, so it is safe to remove.
+        except:
+            pass
+
+    with concurrent.futures.ThreadPoolExecutor(11) as executor:
+        executor.submit(process_git_stats, git_url_titles_preemptions) # One single thread for `process_git_stats()`. Runs concurrently with `process_git_url_title()`.
         for url, title, preemptions, node_pattern in git_url_titles_preemptions:
             executor.submit(process_git_url_title, url, title, preemptions, node_pattern)