// Expirements - Website crawler

var request = require('sync-request')
var fs = require('fs')
var iso_langdetect = require('langdetect').detectOne

function langdetect(text) {
    const languageMap = {
        'en': 'English',
        'ru': 'Russian',
        'it': 'Italian',
        'fr': 'French',
        'de': 'German',
        'es': 'Spanish',
        'pt': 'Portuguese',
        'zh': 'Chinese',
        'ja': 'Japanese',
        'ko': 'Korean',
        'ar': 'Arabic',
        'hi': 'Hindi',
        'pl': 'Polish',
        'nl': 'Dutch',
        'sv': 'Swedish',
        'no': 'Norwegian',
        'da': 'Danish',
        'fi': 'Finnish',
        'tr': 'Turkish',
        'id': 'Indonesian',
        'sl': 'Slovenian'
    };
    var detect = iso_langdetect(text)
    return languageMap[iso_langdetect(text)] || detect;
}

function extractMetadata(url) {
  console.log(`[DEBUG] Receiving metadata for ${url}`)
  const res = request('GET', url);
  const html = res.getBody().toString();

  // TITLE DETECTION (case-insensitive)
  const lower = html.toLowerCase();
  const open = '<title>';
  const close = '</title>';
  const start = lower.indexOf(open);
  if (start === -1) {
    console.warn(`No title detected for ${url}`);
    return;
  }
  const end = lower.indexOf(close, start + open.length);
  if (end === -1) {
    console.warn(`No title detected for ${url}`);
    return;
  }
  const title = html.slice(start + open.length, end).trim();

  // DESCRIPTION DETECTION (no regex, no deps)
  let desc = '';
  const lowerMeta = lower;
  // find a <meta ... name="description" ...> or name='description'
  const namePos = lowerMeta.indexOf('name="description"');
  const namePosAlt = lowerMeta.indexOf("name='description'");
  const nameIndex = namePos !== -1 ? namePos : (namePosAlt !== -1 ? namePosAlt : -1);

  if (nameIndex !== -1) {
    // search backwards to find start of the tag '<meta'
    const tagStart = lowerMeta.lastIndexOf('<meta', nameIndex);
    if (tagStart !== -1) {
      // search for content= after the name attribute position
      const contentAttrPos = lowerMeta.indexOf('content=', nameIndex);
      if (contentAttrPos !== -1) {
        // determine quote char used for content value
        const quoteChar = html[contentAttrPos + 8]; // position right after content=
        const possibleQuote = quoteChar === '"' || quoteChar === "'" ? quoteChar : null;
        if (possibleQuote) {
          const valueStart = contentAttrPos + 9; // after content=" or content='
          const valueEnd = html.indexOf(possibleQuote, valueStart);
          if (valueEnd !== -1) {
            desc = html.slice(valueStart, valueEnd).trim();
          }
        } else {
          // unquoted content= value — read until space or '>'
          const valueStart = contentAttrPos + 8;
          const rest = html.slice(valueStart);
          const m = rest.match(/^[^\s>]+/);
          if (m) desc = m[0].trim();
        }
      } else {
        // sometimes attributes order is reversed: look for content= before the name attribute within the tag
        const contentBeforePos = lowerMeta.lastIndexOf('content=', nameIndex);
        if (contentBeforePos !== -1 && contentBeforePos > tagStart) {
          const quoteChar = html[contentBeforePos + 8];
          const possibleQuote = quoteChar === '"' || quoteChar === "'" ? quoteChar : null;
          if (possibleQuote) {
            const valueStart = contentBeforePos + 9;
            const valueEnd = html.indexOf(possibleQuote, valueStart);
            if (valueEnd !== -1) {
              desc = html.slice(valueStart, valueEnd).trim();
            }
          }
        }
      }
    }
  }
  //

  if (title) return [url, title, desc, langdetect(title + desc)];
  console.warn(`No title detected for ${url}`);
}

function crawlWebsite(url, limit=100) {
    // Array to store unique URLs
    const uniqueLinks = [];

    // Function to get links from a given URL
    function getLinks(currentUrl) {
        try {
            // Sending a synchronous request to the URL
            const res = request('GET', currentUrl);
            const contentType = res.headers['content-type'];

            // Check if the content type contains 'text/html'
            if (contentType && contentType.includes('text/html')) {
                const html = res.getBody('utf8');

                // Regex to find all URLs in <a> tags
                const regex = /<a[^>]+href="([^">]+)"/g;
                let match;

                while ((match = regex.exec(html)) !== null) {
                    const href = match[1];
                    // Resolve relative URLs
                    const absoluteUrl = new URL(href, currentUrl).href;

                    if (!uniqueLinks.includes(absoluteUrl)) {
                        uniqueLinks.push(absoluteUrl);
                        console.log(`[DEBUG] Crawling ${absoluteUrl} (${uniqueLinks.length})`);
                        // Recursive call to get links only if the URL is unique
                        if (uniqueLinks.length < (limit / 2)) {
                            getLinks(absoluteUrl);
                        } else {
                            return uniqueLinks
                        }
                    }
                }
            } else {
                console.log(`[INFO] Skipping non-HTML content at ${currentUrl}`);
            }

        } catch (error) {
            console.error(`Error crawling ${currentUrl}: ${error.message}`);
        }
    }

    // Start the crawling process
    getLinks(url);

    // Return the array of unique links
    return uniqueLinks;
}

function makeSearchDB(startpoint, entries, outfile) {
    var crawling = crawlWebsite(startpoint, entries).map((url) => {
        return extractMetadata(url) || [url, '', '', 'Unknown']
    })

    var csv = []
    crawling.forEach((entry) => {
        csv.push(entry.join(','))
    })
    fs.writeFileSync(outfile, csv.join("\n"))
}

makeSearchDB("http://cows.info.gf:90", 5, "search.csv")

/*
           DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
                   Version 2, December 2004

Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>

Everyone is permitted to copy and distribute verbatim or modified
copies of this license document, and changing it is allowed as long
as the name is changed.

           DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
  TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION

 0. You just DO WHAT THE FUCK YOU WANT TO.
*/