// index.js

// including environment variables in .env
require('dotenv').config();

const cookieManager = require('./src/helpers/cookie-manager');
const { setLocalCookie } = require('./src/helpers/cookie-storage');

// IIFE - application entry point
(async () => {
  // CookieManager call point
  // login/password values are stored in the .env file
  const cookie = await cookieManager.fetchCookie(
    process.env.LOGIN,
    process.env.PASSWORD,
  );

  if (cookie) {
    // if the cookie was received, assign it as the value of a storage variable
    setLocalCookie(cookie);
  } else {
    console.log('Warning! Could not fetch the Cookie after 3 attempts. Aborting the process...');
    // close the application with an error if it is impossible to receive the cookie
    process.exit(1);
  }
})();
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.

// cookie-manager.js

// 'user-agents' generates 'User-Agent' values for HTTP headers
// 'puppeteer-extra' - wrapper for 'puppeteer' library
const _ = require('lodash');
const UserAgent = require('user-agents');
const puppeteerXtra = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');

// hide from webserver that it is bot
puppeteerXtra.use(StealthPlugin());

class CookieManager {
  // this.browser & this.page - Chromium window and page instances 
  constructor() {
    this.browser = null;
    this.page = null;
    this.cookie = null;
  }

  // getter
  getCookie() {
    return this.cookie;
  }

  // setter
  setCookie(cookie) {
    this.cookie = cookie;
  }

  async fetchCookie(username, password) {
    // give 3 attempts to authorize and receive cookies
    const attemptCount = 3;

    try {
      // instantiate Chromium window and blank page 
      this.browser = await puppeteerXtra.launch({
        args: ['--window-size=1920,1080'],
        headless: process.env.NODE_ENV === 'PROD',
      });

      // Chromium instantiates blank page and sets 'User-Agent' header
      this.page = await this.browser.newPage();
      await this.page.setUserAgent((new UserAgent()).toString());

      for (let i = 0; i < attemptCount; i += 1) {
        // Chromium asks the web server for an authorization page
        //and waiting for DOM
        await this.page.goto(process.env.LOGIN_PAGE, { waitUntil: ['domcontentloaded'] });

        // Chromium waits and presses the country selection confirmation button
        // and falling asleep for 1 second: page.waitForTimeout(1000)
        await this.page.waitForSelector('#changeRegionAndLanguageBtn', { timeout: 5000 });
        await this.page.click('#changeRegionAndLanguageBtn');
        await this.page.waitForTimeout(1000);

        // Chromium waits for a block to enter a username and password
        await this.page.waitForSelector('div.login-box-content', { timeout: 5000 });
        await this.page.waitForTimeout(1000);
        
        // Chromium enters username/password and clicks on the 'Log in' button      
        await this.page.type('input.email-input', username);
        await this.page.waitForTimeout(1000);
        await this.page.type('input.password-input', password);
        await this.page.waitForTimeout(1000);
        await this.page.click('button[value="Log in"]');
        await this.page.waitForTimeout(3000);

        // Chromium waits for target content to load on 'div.main' selector
        await this.page.waitForSelector('div.main', { timeout: 5000 });

        // get the cookies and glue them into a string of the form <key>=<value> [; <key>=<value>]
        this.setCookie(
          _.join(
            _.map(
              await this.page.cookies(),
              ({ name, value }) => _.join([name, value], '='),
            ),
            '; ',
          ),
        );

        // when the cookie has been received, break the loop
        if (this.cookie) break;
      }

      // return cookie to call point (in index.js)
      return this.getCookie();
    } catch (err) {
      throw new Error(err);
    } finally {
      // close page and browser instances
      this.page && await this.page.close();
      this.browser && await this.browser.close();
    }
  }
}

// export singleton
module.exports = new CookieManager();
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.
59.
60.
61.
62.
63.
64.
65.
66.
67.
68.
69.
70.
71.
72.
73.
74.
75.
76.
77.
78.
79.
80.
81.
82.
83.
84.
85.
86.
87.
88.
89.
90.
91.
92.
93.
94.
95.
96.
97.
98.
99.
100.

// cookie-storage.js

//  cookie storage variable
let localProcessedCookie = null;

// getter
const getLocalCookie = () => localProcessedCookie;

// setter
const setLocalCookie = (cookie) => {
  localProcessedCookie = cookie;

  // lock the getLocalCookie function; 
  // its scope with the localProcessedCookie value will be saved
  // after the setLocalCookie function completes
  return getLocalCookie;
};

module.exports = {
  setLocalCookie,
  getLocalCookie,
};
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.

// index.js

// import builder function
const getTargetUrls = require('./src/helpers/target-builder');

(async () => {
  // here the proccess of getting cookie

  // gets an array of url links and determines it's length L
  const targetUrls = getTargetUrls();
  const { length: L } = targetUrls;

})();
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.

// categories.js

module.exports = [
  'women/clothing',
  'women/footwear',
  'women/accessories/all-womens-accessories',
  'men/clothing',
  'men/footwear',
  'men/accessories/all-mens-accessories',
  'kids/girls/clothing/all-girls-clothing',
  'kids/boys/clothing/all-boys-clothing',
];
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.

// target-builder.js

const path = require('path');
const categories = require('../constants/categories');

// static fragment of route parameters
const staticPath = 'global/en';

// create URL object from main page address
const url = new URL(process.env.MAIN_PAGE);

// add the full string of route parameters to the URL object
// and return full url string
const addPath = (dynamicPath) => {
  url.pathname = path.join(staticPath, dynamicPath);

  return url.href;
};

// collect URL link from each element of the array with categories
module.exports = () => categories.map((category) => addPath(category));
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.

// index.js

const crawler = new Crawler();

(async () => {
  // getting Cookie proccess
  // and url-links array...
  const { length: L } = targetUrls;

  // run a loop through the length of the array of url links
  for (let i = 0; i < L; i += 1) {
    // call the run method of the crawler for each link
    // and return parsed data
    const result = await crawler.run(targetUrls[i]);

    // do smth with parsed data...
  }
})();
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.

// crawler.js

require('dotenv').config();
const cheerio = require('cheerio');
const axios = require('axios').default;
const UserAgent = require('user-agents');

const Parser = require('./parser');
// getLocalCookie - closure function, returns localProcessedCookie
const { getLocalCookie } = require('./helpers/cookie-storage');

module.exports = class Crawler {
  constructor() {
    // create a class variable and bind it to the newly created Axios object
    // with the necessary headers
    this.axios = axios.create({
      headers: {
        cookie: getLocalCookie(),
        'user-agent': (new UserAgent()).toString(),
      },
    });
  }

  async run(url) {
    console.log('IScraper: working on %s', url);

    try {
      // do HTTP request to the web server
      const { data } = await this.axios.get(url);
      // create a cheerio object with nodes from html markup
      const $ = cheerio.load(data);

      // if the cheerio object contains nodes, run Parser
      // and return to index.js the result of parsing
      if ($.length) {
        const p = new Parser($);

        return p.parse();
      }
      console.log('IScraper: could not fetch or handle the page content from %s', url);
      return null;
    } catch (e) {
      console.log('IScraper: could not fetch the page content from %s', url);

      return null;
    }
  }
};
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.

[
  {
    "Title":"Graphic relaxed-fit T-shirt | Women",
    "CurrentPrice":25.96,
    "Currency":"€",
    "isNew":false
  },
  {
    // at all 36 such elements for every url-link
  }
] 
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.

// parser.js

require('dotenv').config();
const _ = require('lodash');

module.exports = class Parser {
  constructor(content) {
    // this.$ - this is a cheerio object parsed from the markup
    this.$ = content;
    this.$$ = null;
  }

  // The crawler calls the parse method
  // extracts all 'li' elements from the content block
  // and in the map loop the target data is selected
  parse() {
    return this.$('#js-search-result-items')
      .children('li')
      .map((i, el) => {
        this.$$ = this.$(el);

        const Title = this.getTitle();
        const CurrentPrice = this.getCurrentPrice();

        // if two key values are missing, such object is rejected
        if (!Title || !CurrentPrice) return {};

        return {
          Title,
          CurrentPrice,
          Currency: this.getCurrency(),
          isNew: this.isNew(),
        };
      })
      .toArray();
  }

  // next - private methods, which are used at 'parse' method
  getTitle() {
    return _.replace(this.$$.find('.product__name').text().trim(), /\s{2,}/g, ' ');
  }

  getCurrentPrice() {
    return _.toNumber(
      _.replace(
        _.last(_.split(this.$$.find('.product__price').text().trim(), ' ')),
        ',',
        '.',
      ),
    );
  }

  getCurrency() {
    return _.head(_.split(this.$$.find('.product__price').text().trim(), ' '));
  }

  isNew() {
    return /new/.test(_.toLower(this.$$.find('.content-asset p').text().trim()));
  }
};
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.
59.
60.

myServer.on('request', (request, response) => {
  // something puts into response
});

// or

myObject.on('data', (chunk) => {
  // do something with data
});
1.
2.
3.
4.
5.
6.
7.
8.
9.

// index.js

const fs = require('fs');
const path = require('path');
const { createWriteStream } = require('fs');

// use constants to simplify work with addresses
const resultDirPath = path.join('./', 'data');
const resultFilePath = path.join(resultDirPath, 'data.json');

// check if the data directory exists; create if it's necessary
// if the data.json file existed - delete all data
//  ...if not existed - create empty
!fs.existsSync(resultDirPath) && fs.mkdirSync(resultDirPath);
fs.writeFileSync(resultFilePath, '');

(async () => {
  // getting Cookie proccess
  // and url-links array...

  // create a stream object for writing
  // and add square bracket to the first line with a line break
  const writer = createWriteStream(resultFilePath);
  writer.write('[\n');

  // run a loop through the length of the url-links array 
  for (let i = 0; i < L; i += 1) {
    const result = await crawler.run(targetUrls[i]);

    // if an array with parsed data is received, determine its length l
    if (!_.isEmpty(result)) {
      const { length: l } = result;

      // using the write method, add the next portion 
      //of the incoming data to data.json
      for (let j = 0; j < l; j += 1) {
        if (i + 1 === L && j + 1 === l) {
          writer.write(`  ${JSON.stringify(result[j])}\n`);
        } else {
          writer.write(`  ${JSON.stringify(result[j])},\n`);
        }
      }
    }
  }
})();
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.

[
  {"Title":"Graphic relaxed-fit T-shirt | Women","CurrentPrice":25.96,"Currency":"€","isNew":false},
  {"Title":"Printed mercerised T-shirt | Women","CurrentPrice":29.97,"Currency":"€","isNew":true},
  {"Title":"Slim-fit camisole | Women","CurrentPrice":32.46,"Currency":"€","isNew":false},
  {"Title":"Graphic relaxed-fit T-shirt | Women","CurrentPrice":25.96,"Currency":"€","isNew":true},
  ...
  {"Title":"Piped-collar polo | Boys","CurrentPrice":23.36,"Currency":"€","isNew":false},
  {"Title":"Denim chino shorts | Boys","CurrentPrice":45.46,"Currency":"€","isNew":false}
]
1.
2.
3.
4.
5.
6.
7.
8.
9.