用Python实现编程语言 20 年的动态排行榜

2020/06/24Python83724325 字10 分钟

爬取一下，自2001年5月至今，TIOBE 编程语言排行榜上编程语言的变化情况，看一下在接近20年的时间里，编程语言的热度是如何变化的。

1、数据来源

https://www.tiobe.com/tiobe-index/

2、数据提取

将原始数据从目标站点获取到，如下所如,站点部分源码

series: [
    	{name : 'C',data : [[Date.UTC(2001, 5, 30), 20.24], [Date.UTC(2001, 6, 30), 20.77], [Date.UTC(2001, 7, 30), 20.75], [Date.UTC(2001, 8, 28), 20.77], [Date.UTC(2001, 9, 26), 19.75], [Date.UTC(2001, 10, 28), 19.21], [Date.UTC(2001, 11, 31), 20.14], [Date.UTC(2002, 0, 30), 18.83], [Date.UTC(2002, 1, 27), 19.89]]}
    ]

解析数据核心代码

#发送请求获取数据
response = requests.get(self.url, headers=self.headers, proxies=self.proxies).text
#获取总的全部数据
total_content = ''.join(re.findall(r'series:(.*?)\}\);', response, re.DOTALL))
#获取每种编程语言的数据
total_content = re.findall(r'({.*?})', total_content, re.DOTALL)
#print(total_content)

with open(self.table, 'w', newline='') as f:
    self.writer = csv.DictWriter(f, ['name', 'value', 'date'])
    self.writer.writeheader()

    for content in total_content:
        #获取编程语言的名字
        name = ''.join(re.findall(r"{name : '(.*?)'", content, re.DOTALL))
        #获取编程语言数据
        data = re.findall(r"\[Date.UTC(.*?)\]", content, re.DOTALL)
        for i in data:
            i = i.replace(' ', '')
            #提取时间
            i = re.sub(r'[()]', '', i)
            value = i.split(',')[-1]
            date_list = i.split(',')[:3]
            time = []
            for index,j in enumerate(date_list):
                if index != 0:
                    if len(j) == 1:
                        j = '0'+j
                time.append(j)
            self.writer.writerow({'name':name, 'value':value, 'date':'-'.join(time)})
            print(name, value, '-'.join(time))

对于抓取到的数据，进行正则表达式匹配，然后提取各个编程语言在不同时间段的热度数值，并保存到本地的文件中。

3、清洗数据

我们要完成的就是利用动态可视化的柱状图来观察各种编程语言随着时间的热度变化。我们先对数据进行清洗，获取编程语言的名字一起设置一个嵌套的字典，程序如下图所示：

data = pd.read_csv(self.table)
#获取编程语言的名字
names = set(data['name'])
#print(names)
#获取编程语言的月份信息
month_list = data[data['name'].isin(['Java'])]['date'].values
#print(month_list)
#创建嵌套列表，每个日期下保留每种编程语言的热度值
per_month_lan_dict = {i:{j:0 for j in names} for i in month_list}
for month in month_list:
    for name in names:
        try:
            per_month_lan_dict[month][name] = data[data['name'] == name][data['date'] == month]['value'].values[0]
        except:
            continue
print(per_month_lan_dict)
return per_month_lan_dict

上述程序中的嵌套字典含义为每一个月份下的每种编程语言的热度值，其结构格式如下所示：

{“2020-1-12”:{“Java”:16, “C++”:14, “python”:10,…}, “2020-2-13”:{“Java”:16.3, “C++”:15.6, …},…}。

4、设置图表

为了在可视化过程中区分每一种编程语言，需要为柱状图中的每一柱都设置不同的颜色，同时，将嵌套字典按照月份的顺序进行排序，程序如下所示：

colors = ['k', 'r', 'sienna', 'yellow', 'g', 'aquamarine', 'dodgerblue', 'pink', 'b', 'darkviolet']
color_dict = {
    'C#':'k',
    'Python':'r',
    'C++':'sienna',
    'JavaScript':'yellow',
    'Visual Basic':'g',
    'R':'aquamarine',
    'C':'dodgerblue',
    'Java':'pink',
    'PHP':'b',
    'SQL':'darkviolet'
}

5、动态显示

最后，我们便可以对数据进行可视化的展示，程序如下图所示：

ax = plt.gca()
for data_item in data.items():
    plt.cla()
    temp = sorted(data_item[1].items(), key=lambda item: item[1])
    x = [item[0] for item in temp]
    color = [color_dict[i] for i in x]
    y = [item[1] for item in temp]
	
    plt.barh(range(1, 11), y, color = color)
    plt.title(data_item[0], fontproperties='simhei', fontsize=16)
    plt.yticks(range(1,11), list(x), fontproperties='simhei', fontsize=16)
    plt.xticks(range(0, 30, 100))
    for x,y in zip(range(1, 11), y):
        plt.text(y + 0.1, x - 0.1, str(y))
    plt.pause(0.1)
plt.show()

6、完整代码

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@File    :   DynamicChart.py
@Time    :   2020/06/22 10:11:51
@Author  :   MrYong 
@Version :   1.0
@Contact :   m.yong@foxmail.com
@License :   (C)Copyright 2018-2019, MR YONG
@Desc    :   做了一个编程语言 20 年的动态排行榜！
'''

# here put the import lib
import requests
import pandas as pd
import re
import csv
from collections import OrderedDict
import matplotlib.pyplot as plt

class Spider():
    def __init__(self):
        self.url = 'https://www.tiobe.com/tiobe-index/'
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
        }
        self.proxies = {
            'http':'http://web-proxy.tencent.com:8080/',
            'https':'http://web-proxy.tencent.com:8080/'
        }
        self.table = './sp_data.csv'
    def run(self):
        #self.getData()
        data = self.analysisData()
        self.draw(data)

    def draw(self, data):
        colors = ['k', 'r', 'sienna', 'yellow', 'g', 'aquamarine', 'dodgerblue', 'pink', 'b', 'darkviolet']
        color_dict = {
            'C#':'k',
            'Python':'r',
            'C++':'sienna',
            'JavaScript':'yellow',
            'Visual Basic':'g',
            'R':'aquamarine',
            'C':'dodgerblue',
            'Java':'pink',
            'PHP':'b',
            'SQL':'darkviolet'
        }

        ax = plt.gca()
        for data_item in data.items():
            plt.cla()

            temp = sorted(data_item[1].items(), key=lambda item: item[1])
            x = [item[0] for item in temp]
            color = [color_dict[i] for i in x]
            y = [item[1] for item in temp]

            plt.barh(range(1, 11), y, color = color)
            plt.title(data_item[0], fontproperties='simhei', fontsize=16)
            plt.yticks(range(1,11), list(x), fontproperties='simhei', fontsize=16)
            plt.xticks(range(0, 30, 100))
            for x,y in zip(range(1, 11), y):
                plt.text(y + 0.1, x - 0.1, str(y))
            plt.pause(0.1)
        plt.show()

    def sort_key(self, old_dict, reverse = False):
        '''
        对字典按key排序，并返回OrderedDict
        '''
        keys = sorted(old_dict.keys(), reverse=reverse)
        #创建新的空字典
        new_dict = OrderedDict()
        for key in keys:
            new_dict[key] = old_dict[key]
        return new_dict

    def analysisData(self):
        data = pd.read_csv(self.table)
        #获取编程语言的名字
        names = set(data['name'])
        #print(names)

        #获取编程语言的月份信息
        month_list = data[data['name'].isin(['Java'])]['date'].values
        #print(month_list)

        #创建嵌套列表，每个日期下保留每种编程语言的热度值
        per_month_lan_dict = {i:{j:0 for j in names} for i in month_list}
        for month in month_list:
            for name in names:
                try:
                    per_month_lan_dict[month][name] = data[data['name'] == name][data['date'] == month]['value'].values[0]
                except:
                    continue
        print(per_month_lan_dict)
        return per_month_lan_dict

    def getData(self):
        #发送请求获取数据
        response = requests.get(self.url, headers=self.headers, proxies=self.proxies).text
        #获取总的全部数据
        total_content = ''.join(re.findall(r'series:(.*?)\}\);', response, re.DOTALL))
        #获取每种编程语言的数据
        total_content = re.findall(r'({.*?})', total_content, re.DOTALL)
        #print(total_content)

        with open(self.table, 'w', newline='') as f:
            self.writer = csv.DictWriter(f, ['name', 'value', 'date'])
            self.writer.writeheader()

            for content in total_content:
                #获取编程语言的名字
                name = ''.join(re.findall(r"{name : '(.*?)'", content, re.DOTALL))
                #获取编程语言数据
                data = re.findall(r"\[Date.UTC(.*?)\]", content, re.DOTALL)

                for i in data:
                    i = i.replace(' ', '')
                    #提取时间
                    i = re.sub(r'[()]', '', i)
                    value = i.split(',')[-1]
                    date_list = i.split(',')[:3]
                    time = []
                    for index,j in enumerate(date_list):
                        if index != 0:
                            if len(j) == 1:
                                j = '0'+j
                        time.append(j)
                    
                    self.writer.writerow({'name':name, 'value':value, 'date':'-'.join(time)})
                    print(name, value, '-'.join(time))

if __name__ == "__main__":
    sp = Spider()
    sp.run()

暂无评论