import os
import urllib.request
from scrapy.selector import Selector
from scrapy.http import HtmlResponse
import scrapy

from scrapy.crawler import CrawlerProcess
from scrapy.settings import Settings

class VSCodeTagSpider(scrapy.Spider):
    name = "vscode_tags"
    allowed_domains = ["visualstudio.com"]
    start_urls = ['https://marketplace.visualstudio.com/search?target=VSCode&category=All%20categories&sortBy=Installs']

    def parse(self, response):
        print('todo')

def fetch():
    settings = Settings()
    process = CrawlerProcess()
    process.crawl(VSCodeTagSpider)
    process.start()