Fix to many files open (#35)

* add semaphore to limit number of files open

* made request async instead of sync

Author: ThatAmatoGuy

deno.json

@@ -25,6 +25,7 @@
         "@cloudydeno/kubernetes-apis": "jsr:@cloudydeno/kubernetes-apis@0.5.2",
         "@cloudydeno/kubernetes-client": "jsr:@cloudydeno/kubernetes-client@0.7.5",
         "@deno-library/compress": "jsr:@deno-library/compress@^0.5.6",
+        "@henrygd/semaphore": "jsr:@henrygd/semaphore@^0.0.2",
         "@std/yaml": "jsr:@std/yaml@1.0.7"
     }
 }

deno.lock

@@ -1,20 +1,33 @@
 import { stringify as toYaml } from '@std/yaml';
 import { tgz } from '@deno-library/compress';
 import { getPodLogs } from './k8s.js';
+import { getSemaphore } from '@henrygd/semaphore';
 
-export function writeYaml(data, name, dirPath) {
-    Deno.mkdirSync(dirPath, { recursive: true });
-    const filePath = `${dirPath}/${name}.yaml`;
-    Deno.writeTextFileSync(filePath, toYaml(data, { skipInvalid: true }));
+const semaphore = getSemaphore('supportPackageSemaphore', 10);
+
+export async function writeYaml(data, name, dirPath) {
+    await semaphore.acquire();
+    try {
+        await Deno.mkdir(dirPath, { recursive: true });
+        const filePath = `${dirPath}/${name}.yaml`;
+        await Deno.writeTextFile(filePath, toYaml(data, { skipInvalid: true }));
+    } finally {
+        semaphore.release();
+    }
 }
 
 export async function preparePackage(dirPath) {
-    const supportPackageZip = `${dirPath}.tar.gz`;
-    console.log('Preparing the Support Package');
-    await tgz.compress(dirPath, supportPackageZip);
-    console.log('Cleaning up temp directory');
-    Deno.removeSync(dirPath, { recursive: true });
-    console.log(`\nPlease attach ${supportPackageZip} to your support ticket.`);
+    await semaphore.acquire();
+    try {
+        const supportPackageZip = `${dirPath}.tar.gz`;
+        console.log('Preparing the Support Package');
+        await tgz.compress(dirPath, supportPackageZip);
+        console.log('Cleaning up temp directory');
+        await Deno.remove(dirPath, { recursive: true });
+        console.log(`\nPlease attach ${supportPackageZip} to your support ticket.`);
+    } finally {
+        semaphore.release();
+    }
 }
 
 export async function processData(dirPath, k8sResources) {
@@ -29,16 +42,21 @@ export async function processData(dirPath, k8sResources) {
 
         if (k8sType == 'pods') {
             for (const pod of resources.items) {
-                delete pod.metadata.managedFields;
+                await semaphore.acquire();
+                try {
+                    delete pod.metadata.managedFields;
 
-                writeYaml(pod, `spec_${pod.metadata.name}`, `${dirPath}/${k8sType}/${pod.metadata.name}`);
+                    await writeYaml(pod, `spec_${pod.metadata.name}`, `${dirPath}/${k8sType}/${pod.metadata.name}`);
 
-                const logs = await getPodLogs(pod);
-                for (const [containerName, logData] of Object.entries(logs)) {
-                    Deno.writeTextFileSync(
-                        `${dirPath}/${k8sType}/${pod.metadata.name}/log_${containerName}.log`,
-                        logData,
-                    );
+                    const logs = await getPodLogs(pod);
+                    for (const [containerName, logData] of Object.entries(logs)) {
+                        await Deno.writeTextFile(
+                            `${dirPath}/${k8sType}/${pod.metadata.name}/log_${containerName}.log`,
+                            logData,
+                        );
+                    }
+                } finally {
+                    semaphore.release();
                 }
             }
             continue;
@@ -60,14 +78,19 @@ export async function processData(dirPath, k8sResources) {
             const header = 'LAST SEEN\tTYPE\tREASON\tOBJECT\tMESSAGE\n';
             const content = header + formattedEvents.join('\n');
 
-            Deno.writeTextFileSync(`${dirPath}/${k8sType}.csv`, content);
+            await Deno.writeTextFile(`${dirPath}/${k8sType}.csv`, content);
 
             continue;
         }
 
-        resources.items.map((data) => {
-            delete data.metadata.managedFields;
-            writeYaml(data, `${data.metadata.name}_get`, `${dirPath}/${k8sType}`);
-        });
+        await Promise.all(resources.items.map(async (data) => {
+            await semaphore.acquire();
+            try {
+                delete data.metadata.managedFields;
+                await writeYaml(data, `${data.metadata.name}_get`, `${dirPath}/${k8sType}`);
+            } finally {
+                semaphore.release();
+            }
+        }));
     }
 }

src/logic/core.js

@@ -27,14 +27,21 @@ export async function onprem(namespace) {
     }
 
     if (cfCreds) {
-        const accounts = await getAllAccounts(cfCreds);
-        writeYaml(accounts, 'OnPrem_Accounts', dirPath);
-        const runtimes = await getAllRuntimes(cfCreds);
-        writeYaml(runtimes, 'OnPrem_Runtimes', dirPath);
-        const featureFlags = await getSystemFeatureFlags(cfCreds);
-        writeYaml(featureFlags, 'OnPrem_Feature_Flags', dirPath);
-        const totalUsers = await getTotalUsers(cfCreds);
-        writeYaml(totalUsers, 'OnPrem_Total_Users', dirPath);
+        const dataFetchers = [
+            { name: 'OnPrem_Accounts', fetcher: getAllAccounts },
+            { name: 'OnPrem_Runtimes', fetcher: getAllRuntimes },
+            { name: 'OnPrem_Feature_Flags', fetcher: getSystemFeatureFlags },
+            { name: 'OnPrem_Total_Users', fetcher: getTotalUsers },
+        ];
+
+        for (const { name, fetcher } of dataFetchers) {
+            try {
+                const data = await fetcher(cfCreds);
+                await writeYaml(data, name, dirPath);
+            } catch (error) {
+                console.error(`Failed to fetch or write ${name}:`, error.message);
+            }
+        }
     }
 
     console.log(`Gathering data in the '${namespace}' namespace for Codefresh OnPrem`);

src/onprem.js

@@ -30,11 +30,11 @@ export async function pipelines(namespace, runtime) {
                 } while (isNaN(selection) || selection < 1 || selection > runtimes.length);
 
                 const reSpec = runtimes[selection - 1];
-                writeYaml(reSpec, 'Runtime_Spec', dirPath);
+                await writeYaml(reSpec, 'Runtime_Spec', dirPath);
             }
         } else {
-            const reSpec = getRuntimeSpec(cfCreds, runtime);
-            writeYaml(reSpec, 'Runtime_Spec', dirPath);
+            const reSpec = await getRuntimeSpec(cfCreds, runtime);
+            await writeYaml(reSpec, 'Runtime_Spec', dirPath);
         }
     }