ini复制代码public int findPoisonedDuration(int[] timeSeries, int duration) {
    int result = 0;
    //遍历所有攻击
    for (int i = 0; i < timeSeries.length - 1; i++) {
        // 得到下一次攻击 和 本次攻击的 之间的时间差值，然后与 中毒持续时间 进行比较
        int item = timeSeries[i + 1] - timeSeries[i];
        // 如果时间差值 小于等于 中毒持续时间，那么下一次攻击时，中毒状态会重置，故本次中毒只会持续到下一次中毒的时候
        if (item <= duration) {
            result = result + item;
        } else {
            //如果时间差值  大于 中毒持续时间，那么本次中毒会持续完整的时间。
            result = result + duration;
        }
    }
    //最后一次攻击,会持续完整时间
    result = result + timeSeries[timeSeries.length - 1];
    return result;
}

bash复制代码curl --user rest-admin:test http://localhost:8080/flowable-rest/service/management/engine

bash复制代码curl --user rest-admin:test -F "file=@holiday-request.bpmn20.xml" http://localhost:8080/flowable-rest/service/repository/deployments

bash复制代码curl --user rest-admin:test http://localhost:8080/flowable-rest/service/repository/process-definitions

bash复制代码curl --user rest-admin:test -H "Content-Type: application/json" -X POST -d '{ "processDefinitionKey":"holidayRequest", "variables": [ { "name":"employee", "value": "John Doe" }, { "name":"nrOfHolidays", "value": 7 }]}' http://localhost:8080/flowable-rest/service/runtime/process-instances

bash复制代码curl --user rest-admin:test -H "Content-Type: application/json" -X POST -d '{ "candidateGroup" : "managers" }' http://localhost:8080/flowable-rest/service/query/tasks

bash复制代码curl --user rest-admin:test -H "Content-Type: application/json" -X POST -d '{ "action" : "complete", "variables" : [ { "name" : "approved", "value" : true} ]  }' http://localhost:8080/flowable-rest/service/runtime/tasks/25

xml复制代码<serviceTask id="externalSystemCall" name="Enter holidays in external system"
                     flowable:class="org.flowable.CallExternalSystemDelegate"/>

java复制代码package org.flowable;

import org.flowable.engine.delegate.DelegateExecution;
import org.flowable.engine.delegate.JavaDelegate;

public class CallExternalSystemDelegate implements JavaDelegate {

    public void execute(DelegateExecution execution) {
        System.out.println("Calling the external system for employee "
                + execution.getVariable("employee"));
    }

}

txt复制代码http://ask.ci123.com/categories/show/2
http://ask.ci123.com/categories/show/3
http://ask.ci123.com/categories/show/4
http://ask.ci123.com/categories/show/{类别ID}

txt复制代码http://ask.ci123.com/categories/show/4/all?p={页码}

python复制代码import requests

url = "http://ask.ci123.com/"

# 抓取分类页面
def get_category():
    res = requests.get("http://ask.ci123.com/")
    print(res.text)


if __name__ == "__main__":
    get_category()

python复制代码requests.get(url="http://ask.ci123.com/")

python复制代码import requests
payload = {'key1': 'value1', 'key2': 'value2'}
res = requests.get(url="http://ask.ci123.com/", params=payload)
print(res.url)

python复制代码import requests
payload = {'key1': 'value1', 'key2': 'value2'}
headers = {
    'user-agent': 'Baiduspider-image+(+http://www.baidu.com/search/spider.htm)'
}
res = requests.get(url="http://ask.ci123.com/",
                   params=payload, headers=headers)
print(res.url)

python复制代码import requests
payload = {'key1': 'value1', 'key2': 'value2'}
headers = {
    'user-agent': 'Baiduspider-image+(+http://www.baidu.com/search/spider.htm)'
}
cookies = dict(my_cookies='nodream')
res = requests.get(url="http://ask.ci123.com/",
                   params=payload, headers=headers, cookies=cookies)
print(res.text)

python复制代码r = requests.get('http://github.com', allow_redirects=False)

python复制代码import requests
payload = {'key1': 'value1', 'key2': 'value2'}
headers = {
    'user-agent': 'Baiduspider-image+(+http://www.baidu.com/search/spider.htm)'
}
cookies = dict(my_cookies='nodream')
res = requests.get(url="http://ask.ci123.com/",
                   params=payload, headers=headers, cookies=cookies, timeout=3)
print(res.text)

python复制代码import requests
import re

url = "http://ask.ci123.com/"
headers = {
    'user-agent': 'Baiduspider-image+(+http://www.baidu.com/search/spider.htm)'
}
# 抓取分类页面
def get_category():
    res = requests.get("http://ask.ci123.com/", headers=headers)
    pattern = re.compile(
        r'<li><a href="/categories/show/(\d+)">', re.S)
    categories_ids = pattern.findall(res.text)
    print(f"获取到的分类ID如下:",categories_ids)

if __name__ == "__main__":
    get_category()

python复制代码# 抓取分类页面
def get_category():
    res = requests.get("http://ask.ci123.com/", headers=headers)
    pattern = re.compile(
        r'<li><a href="/categories/show/(\d+)">', re.S)
    categories_ids = pattern.findall(res.text)
    print(f"获取到的分类ID如下:", categories_ids)
    for cate in categories_ids:
		# 下述代码中有get_list()函数对应的代码
        get_list(cate)
        time.sleep(1)

python复制代码def get_list(cate):
    # 获取总页码，循环抓取所有页

    res = requests.get(
        f"http://ask.ci123.com/categories/show/{cate}", headers=headers)

    pattern = re.compile(
        r'<a class="list_last_page" href="/categories/show/\d+/all\?p=(\d+)"', re.S)
    totle = pattern.search(res.text).group(1)
    for page in range(1, int(totle)):
        print(f"http://ask.ci123.com/categories/show/{cate}/all?p={page}")
        time.sleep(0.2)

python复制代码import requests
import re
import time

url = "http://ask.ci123.com/"
headers = {
    'user-agent': 'Baiduspider-image+(+http://www.baidu.com/search/spider.htm)'
}


def get_detail(text):
    # 该函数实现解析页面数据,之后存储数据
    pattern = re.compile(r'<li>[.\s]*<a href="/questions/show/(\d+)/" title="(.*?)" class="list_title" target="_blank" >.*?</a>\s*<span class="list_asw">(\d+)<font>.*?</font></span>\s*<a class="list_author"  href="/users/show/\d+" title=".*?">(.*?)</a>\s*<span class="list_time">(.*?)</span>\s*</li>')
    data = pattern.findall(text)
    print(data)
    # 数据存储代码不在编写


def get_list(cate):
    # 获取总页码，循环抓取所有页

    res = requests.get(
        f"http://ask.ci123.com/categories/show/{cate}", headers=headers)

    pattern = re.compile(
        r'<a class="list_last_page" href="/categories/show/\d+/all\?p=(\d+)"', re.S)
    totle = pattern.search(res.text).group(1)
    for page in range(1, int(totle)):
        print(f"http://ask.ci123.com/categories/show/{cate}/all?p={page}")
        res = requests.get(
            f"http://ask.ci123.com/categories/show/{cate}/all?p={page}", headers=headers)

        time.sleep(0.2)
        # 调取列表页数据提取函数
        get_detail(res.text)


# 抓取分类页面
def get_category():
    res = requests.get("http://ask.ci123.com/", headers=headers)
    pattern = re.compile(
        r'<li><a href="/categories/show/(\d+)">', re.S)
    categories_ids = pattern.findall(res.text)
    print(f"获取到的分类ID如下:", categories_ids)
    for cate in categories_ids:
        get_list(cate)
        time.sleep(1)


if __name__ == "__main__":
    get_category()

shell复制代码2021-08-08 18:04:14.721 INFO 12402 --- [ main] i.r.s.StructuredLoggingApplication : Started StructuredLoggingApplication in 0.395 seconds (JVM running for 0.552)

json复制代码{
    "timestamp": "2021-08-08 18:04:14.721",
    "level": "INFO",
    "logger": "io.reflectoring....StructuredLoggingApplication",
    "thread": "main",
    "message": "Started StructuredLoggingApplication ..."
}

java复制代码public class LoggingInterceptor implements HandlerInterceptor {


    @Override
    public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler)
            throws Exception {


        if (request.getHeader("X-CUSTOM-HEADER") != null) {
            MDC.put("codePath", "3rdParty");
        } else {
            MDC.put("codePath", "user");
        }


        return true;
    }


    @Override
    public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler,
            ModelAndView modelAndView) {
        MDC.remove("codePath");
    }
}

java复制代码@Componentpublic
class WebConfigurer implements WebMvcConfigurer {


    @Override
    public void addInterceptors(InterceptorRegistry registry) {
        registry.addInterceptor(new LoggingInterceptor());
    }
}

java复制代码@Componentpublic
class Timer {


    private final DomainService domainService;


    private static final Logger logger = LoggerFactory.getLogger(Timer.class);


    public Timer(DomainService domainService) {
        this.domainService = domainService;
    }


    @Scheduled(fixedDelay = 5000)
    void scheduledHello() {
        MDC.put("codePath", "timer");
        logger.info("log event from timer");
        // do some actual work
        MDC.remove("codePath");
    }


}

java复制代码public class LoggingInterceptor implements HandlerInterceptor {


    @Override
    public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler)
            throws Exception {


        Object principal = SecurityContextHolder.getContext().getAuthentication().getPrincipal();


        if (principal instanceof UserDetails) {
            String username = ((UserDetails) principal).getUsername();
            MDC.put("username", username);
        } else {
            String username = principal.toString();
            MDC.put("username", username);
        }


        return true;
    }


    @Override
    public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler,
            ModelAndView modelAndView) {
        MDC.remove("username");
    }
}

java复制代码@ControllerAdvicepublic
class WebExceptionHandler {


    private static final Logger logger = LoggerFactory.getLogger(WebExceptionHandler.class);


    @ExceptionHandler(Exception.class)
    @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
    public void internalServerError(Exception e) {
        MDC.put("rootCause", getRootCause(e).getClass().getName());
        logger.error("returning 500 (internal server error).", e);
        MDC.remove("rootCause");
    }


    private Throwable getRootCause(Exception e) {
        Throwable rootCause = e;
        while (e.getCause() != null && rootCause.getCause() != rootCause) {
            rootCause = e.getCause();
        }
        return rootCause;
    }


}

xml复制代码<dependencyManagement>
  <dependencies>
    <dependency>
      <groupId>org.springframework.cloud</groupId>
      <artifactId>spring-cloud-dependencies</artifactId>
      <version>2020.0.3</version>
      <type>pom</type>
      <scope>import</scope>
    </dependency>
  </dependencies>
</dependencyManagement><dependencies>
  <dependency>
    <groupId>org.springframework.cloud</groupId>
    <artifactId>spring-cloud-starter-sleuth</artifactId>
  </dependency>
</dependencies>

java复制代码void callThirdPartyService() throws InterruptedException {
    logger.info("log event from the domain service");
    Instant start=Instant.now();
    Thread.sleep(2000); // simulating an expensive operation
    Duration duration=Duration.between(start,Instant.now());
    MDC.put("thirdPartyCallDuration",String.valueOf(duration.getNano()));
    logger.info("call to third-party service successful!");
    MDC.remove("thirdPartyCallDuration");
}

lua复制代码__exists__: "rootCause"

lua复制代码job_status: "ERROR"

lua复制代码username: "user" AND _exists_: "rootCause"

go复制代码type Pool struct {
    work chan func()   // 任务
    sem  chan struct{} // 数量
}

go复制代码func New(size int) *Pool {
    return &Pool{
        work: make(chan func()),
        sem:  make(chan struct{}, size),
    }
}

go复制代码func (p *Pool) NewTask(task func()) {
    select {
        case p.work <- task:
        case p.sem <- struct{}{}:
            go p.worker(task)
    }
}

scss复制代码func (p *Pool) worker(task func()) {
    defer func() { <-p.sem }()
    for {
        task()
        task = <-p.work
    }
}

go复制代码func main()  {
    pool := New(128)
    pool.NewTask(func(){
        fmt.Println("run task")
    })
}

css复制代码func main()  {
    pool := New(2)

    for i := 1; i <5; i++{
        pool.NewTask(func(){
            time.Sleep(2 * time.Second)
            fmt.Println(time.Now())
        })
    }

    // 保证所有的协程都执行完毕
    time.Sleep(5 * time.Second)
}

ini复制代码2020-05-24 23:18:02.014487 +0800 CST m=+2.005207182
2020-05-24 23:18:02.014524 +0800 CST m=+2.005243650
2020-05-24 23:18:04.019755 +0800 CST m=+4.010435443
2020-05-24 23:18:04.019819 +0800 CST m=+4.010499440

bash复制代码# publishInfo
exchange: amq.direct
immediate: false
mandatory: false
routingKey: test

# headerBody
bodySize: 1024
properties: 
    - contentType: 
    - encoding: 
    - deliveryMode: 
    - priority: 
    - correlationId: 
    - replyTo: 
    - expiration: 
    - messageId: 
    - timestamp: 
    - type: 
    - userId: 
    - appId: 
    - clusterId: 
    - headers: {}

# contentBody
二进制消息体bytes

python复制代码#!/usr/bin/python3
# -*- coding: utf-8 -*-
"""
Created on Fri Aug 14 17:47:47 2020: 2021/3/30 上午1:13
@Author  : liudong
@Software: PyCharm
"""

import requests
import re
from copyheaders import headers_raw_to_dict
from bs4 import BeautifulSoup
import pandas as pd


# 根据url和参数获取网页的HTML：

def get_html(url, params):

    my_headers = b'''
    Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
    Accept-Encoding: gzip, deflate, br
    Accept-Language: zh-CN,zh;q=0.9
    Cache-Control: max-age=0
    Connection: keep-alive
    Cookie: x-zp-client-id=448f2b96-6b3a-48e3-e912-e6c8dd73e6cb; sts_deviceid=178832cf3f2680-0b20242883a4a9-6618207c-1296000-178832cf3f3780; Hm_lvt_38ba284938d5eddca645bb5e02a02006=1624877846; urlfrom2=121114584; adfcid2=www.google.com; adfbid2=0; FSSBBIl1UgzbN7NO=5QbLj2_L5kKhv8gnuJa.E1._8RKksG1y5Nt4FRrSajQ7PKGJ8CcWopqTuOLay__ida1esO2ud4AdXKKDI69j9UA; locationInfo_search={%22code%22:%22538%22%2C%22name%22:%22%E4%B8%8A%E6%B5%B7%22%2C%22message%22:%22%E5%8C%B9%E9%85%8D%E5%88%B0%E5%B8%82%E7%BA%A7%E7%BC%96%E7%A0%81%22}; selectCity_search=538; ssxmod_itna=7qGxnDRG0=KGqAKGHKiQRSDQwKfkKqYteb87Dla=xA5D8D6DQeGTb0NpYeYietdigMWPqKYG4iteiFlYfPtb+4OEdD84i7DKqibDCqD1D3qDkbCYxiinDCeDIDWeDiDG+8D0hXl7DjQNXZKkULfBNDz4X2/4UgQDDHfG024dLRIqIgFA+5HYbDbxp9DB6rxBQ/Iqj6znUDgMTTibwbj8DoGiP=fifwn7Dq0YoYCA44fDx=bb4ee2hso7DYFDqojR8DG4xL2iD===; ssxmod_itna2=7qGxnDRG0=KGqAKGHKiQRSDQwKfkKqYteb8D61Fgj40y4rP03aKenjt6D6QMTiBeG2Yn408DewGD; urlfrom=121114584; adfcid=www.google.com; adfbid=0; sts_sg=1; sts_chnlsid=Unknown; zp_src_url=https%3A%2F%2Fwww.google.com.hk%2F; LastCity=%E4%B8%8A%E6%B5%B7; LastCity%5Fid=538; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221071739258%22%2C%22first_id%22%3A%22178832cf3bd20f-0be4af1633ae3d-6618207c-1296000-178832cf3be4b8%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E8%87%AA%E7%84%B6%E6%90%9C%E7%B4%A2%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC%22%2C%22%24latest_referrer%22%3A%22https%3A%2F%2Fwww.google.com%2F%22%7D%2C%22%24device_id%22%3A%22178832cf3bd20f-0be4af1633ae3d-6618207c-1296000-178832cf3be4b8%22%7D; acw_tc=276082a716357771539376802e2983bc8a5c6ad6d09a856f4d30d3892a3cd8; 1420ba6bb40c9512e9642a1f8c243891=68d62e0e-9c02-4c51-b5be-268470d6b21e; d4d6cd0b4a19fa72b8cc377185129bb7=1e7229ad-ee24-4063-9e4b-6522acfeefc7; at=01e2bf60daa14c479e524b22cfaf306f; rt=0747ac22bd424c8da3c28cb6bbd7a8f6; zpfe_probe_token=3d5af381s32ee94285a4e785bfcdba4df809; FSSBBIl1UgzbN7NP=53Ud_uDmd57aqqqmZC5Xn3qKkeoR73_UtjtoQEvodODN_.CWXzEhTjq8aUd0_FtFCmJ7zHbxzlDmsdsmVKETzSt0C8oeOyH7oQmVQMzAfCehTWeQ6QfajFpiObY8ukPfhc73vMi1pSbFiE4Iu4rGZjz8L_8Ww80.iFXTkrYYJ.C4nZ1OPCmdGhgVIZBVau1P0P1.qTYIvWuWSQyPdlNvBFfVCjF4x0XIP4AL9VK0E4YZZzV54JqXOXzFr6ox5zzXRW4NTRXe_iYnJ0B7XRWx07n
    Host: sou.zhaopin.com
    Referer: https://sou.zhaopin.com/
    sec-ch-ua: "Google Chrome";v="95", "Chromium";v="95", ";Not A Brand";v="99"
    sec-ch-ua-mobile: ?0
    sec-ch-ua-platform: "macOS"
    Sec-Fetch-Dest: document
    Sec-Fetch-Mode: navigate
    Sec-Fetch-Site: same-origin
    Sec-Fetch-User: ?1
    Upgrade-Insecure-Requests: 1
    User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36
    '''
    my_headers = headers_raw_to_dict(my_headers)  # 把复制的浏览器需求头转化为字典形式
    req = requests.get(url, headers=my_headers, params=params)
    req.encoding = req.apparent_encoding
    html = req.text

    return html


# 输入url和城市编号，获取由所有职位信息的html标签的字符串组成的列表：

def get_html_list(url, city_num):

    html_list = list()

    for i in range(1, 12):
        params = {'jl': str(city_num), 'kw': '数据分析师', 'p': str(i)}
        html = get_html(url, params)
        soup = BeautifulSoup(html, 'html.parser')
        html_list += soup.find_all(name='a', attrs={'class': 'joblist-box__iteminfo iteminfo'})

    for i in range(len(html_list)):
        html_list[i] = str(html_list[i])

    return html_list


# 根据上面的HTML标签列表，把每个职位信息的有效数据提取出来，保存csv文件：

def get_csv(html_list):

    # city = position = company_name = company_size = company_type = salary = education = ability = experience = evaluation = list()  #
    # 上面赋值方法在这里是错误的，它会让每个变量指向同一内存地址，如果改变其中一个变量，其他变量会同时发生改变

    # table = pd.DataFrame(columns = ['城市','职位名称','公司名称','公司规模','公司类型','薪资','学历要求','技能要求','工作经验要求'])
    city, position, company_name, company_size, company_type, salary, education, ability, experience = ([] for i in range(9))  # 多变量一次赋值

    for i in html_list:

        if re.search(
                '<li class="iteminfo__line2__jobdesc__demand__item">(.*?)</li> <li class="iteminfo__line2__jobdesc__demand__item">(.*?)</li> <li class="iteminfo__line2__jobdesc__demand__item">(.*?)</li>',
                i):
            s = re.search(
                '<li class="iteminfo__line2__jobdesc__demand__item">(.*?)</li> <li class="iteminfo__line2__jobdesc__demand__item">(.*?)</li> <li class="iteminfo__line2__jobdesc__demand__item">(.*?)</li>',
                i).group(1)
            city.append(s)
            s = re.search(
                '<li class="iteminfo__line2__jobdesc__demand__item">(.*?)</li> <li class="iteminfo__line2__jobdesc__demand__item">(.*?)</li> <li class="iteminfo__line2__jobdesc__demand__item">(.*?)</li>',
                i).group(2)
            experience.append(s)
            s = re.search(
                '<li class="iteminfo__line2__jobdesc__demand__item">(.*?)</li> <li class="iteminfo__line2__jobdesc__demand__item">(.*?)</li> <li class="iteminfo__line2__jobdesc__demand__item">(.*?)</li>',
                i).group(3)
            education.append(s)
        else:
            city.append(' ')
            experience.append(' ')
            education.append(' ')


        if re.search('<span class="iteminfo__line1__jobname__name" title="(.*?)">', i):
            s = re.search('<span class="iteminfo__line1__jobname__name" title="(.*?)">', i).group(1)
            position.append(s)
        else:
            position.append(' ')

        if re.search('<span class="iteminfo__line1__compname__name" title="(.*?)">', i):
            s = re.search('<span class="iteminfo__line1__compname__name" title="(.*?)">', i).group(1)
            company_name.append(s)
        else:
            company_name.append(' ')

        if re.search(
                '<span class="iteminfo__line2__compdesc__item">(.*?) </span> <span class="iteminfo__line2__compdesc__item">(.*?) </span>',
                i):
            s = re.search(
                '<span class="iteminfo__line2__compdesc__item">(.*?) </span> <span class="iteminfo__line2__compdesc__item">(.*?) </span>',
                i).group(1)
            company_type.append(s)
            s = re.search(
                '<span class="iteminfo__line2__compdesc__item">(.*?) </span> <span class="iteminfo__line2__compdesc__item">(.*?) </span>',
                i).group(2)
            company_size.append(s)
        else:
            company_type.append(' ')
            company_size.append(' ')

        if re.search('<p class="iteminfo__line2__jobdesc__salary">([\s\S]*?)<', i):
            s = re.search('<p class="iteminfo__line2__jobdesc__salary">([\s\S]*?)<', i).group(1)
            s = s.strip()
            salary.append(s)
        else:
            salary.append(' ')

        s = str()
        l = re.findall('<div class="iteminfo__line3__welfare__item">.*?</div>', i)
        for i in l:
            s = s + re.search('<div class="iteminfo__line3__welfare__item">(.*?)</div>', i).group(1) + ' '
        ability.append(s)

    table = list(zip(city, position, company_name, company_size, company_type, salary, education, ability, experience))

    return table



if __name__ == '__main__':

    url = 'https://sou.zhaopin.com/'
    citys = {'上海':538, '北京':530, '广州':763, '深圳':765, '天津':531, '武汉':736, '西安':854, '成都':801, '南京':635, '杭州':653, '重庆':551, '厦门':682}
    for i in citys.keys():
        html_list = get_html_list(url, citys[i])
        table = get_csv(html_list)
        df = pd.DataFrame(table, columns=['city', 'position', 'company_name', 'company_size', 'company_type', 'salary',
                                          'education', 'ability', 'experience'])
        file_name = i + '.csv'
        df.to_csv(file_name)

python复制代码#!/usr/bin/python3
# -*- coding: utf-8 -*-
"""
Created on Fri Aug 14 17:47:47 2020: 2021/4/2 上午1:30
@Author  : liudong
@Software: PyCharm
"""


import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
plt.rcParams['font.sans-serif'] = ['Heiti TC']    # 指定默认字体：解决plot不能显示中文问题
plt.rcParams['axes.unicode_minus'] = False           # 解决保存图像是负号'-'显示为方块的问题
import re
import os
import seaborn as sns
from wordcloud import WordCloud


citys = ['上海', '北京', '广州', '深圳', '天津', '武汉', '西安', '成都', '南京', '杭州', '重庆', '厦门']


#数据清洗：

def data_clear():

    for i in citys:

        file_name = './' + i + '.csv'
        df = pd.read_csv(file_name, index_col = 0)

        for i in range(0, df.shape[0]):

            s = df.loc[[i],['salary']].values.tolist()[0][0]

            if re.search('(.*)-(.*)',s):
                a = re.search('(.*)-(.*)', s).group(1)
                if a[-1] == '千':
                    a = eval(a[0:-1]) * 1000
                elif a[-1] == '万':
                    a = eval(a[0:-1]) * 10000
                b = re.search('(.*)-(.*)', s).group(2)
                if b[-1] == '千':
                    b = eval(b[0:-1]) * 1000
                elif b[-1] == '万':
                    b = eval(b[0:-1]) * 10000
                s = (a + b) / 2
                df.loc[[i], ['salary']] = s
            else:
                df.loc[[i], ['salary']] = ''

        os.remove(file_name)
        df.to_csv(file_name)



#各个城市数据分析职位数量条形图:

def citys_jobs():

    job_num = list()
    for i in citys:
        file_name = './' + i + '.csv'
        df = pd.read_csv(file_name, index_col = 0)
        job_num.append(df.shape[0])
    df = pd.DataFrame(list(zip(citys, job_num)))
    df = df.sort_values(1, ascending = False)
    x = list(df[0])
    y = list(df[1])

    fig = plt.figure(dpi=200)
    ax = fig.add_axes([0.1, 0.1, 0.8, 0.8])
    ax.bar(x,y,alpha = 0.8)
    ax.set_title('数据分析职位在全国主要城市的数量分布')
    ax.set_ylim(0,350)

    plt.savefig('./数据分析职位在全国主要城市的数量分布.jpg')
    plt.show()


#不同城市薪资分布条形图：

def citys_salary():

    y = list()
    x = citys

    for i in citys:
        file_name = './' + i + '.csv'
        df = pd.read_csv(file_name, index_col=0)
        y0 = df['salary'].mean()
        y.append(round(y0/1000, 1))

    df = pd.DataFrame(list(zip(x,y)))
    df = df.sort_values(1, ascending = False)
    x = list(df[0])
    y = list(df[1])

    fig = plt.figure(dpi=200)
    ax = fig.add_axes([0.1, 0.1, 0.8, 0.8])
    ax.bar(x, y, alpha = 0.8)
    ax.set_title('数据分析职位在一些主要城市的薪资分布（单位：千）')
    ax.set_ylim(5, 18)
    for a, b, label in zip(x, y, y):  # 内置函数zip()：将几个列表合并为二维列表并转置，返回一个特殊对象，可通过list()列表化之后查看
        plt.text(a, b, label, horizontalalignment = 'center', fontsize = 10)  # plt.text()函数：在图中（a,b)位置添加一个文字标签label

    plt.savefig('./数据分析职位在一些主要城市的薪资分布.jpg')
    plt.show()


#数据分析岗位总体薪资的分布

def salary_distribute():

    salary_list = list()
    for i in citys:
        file_name = './' + i + '.csv'
        df = pd.read_csv(file_name, index_col = 0)
        salary_list += list(df['salary'])
    salarys = list()
    for i in range(len(salary_list)):
        if not pd.isnull(salary_list[i]):   #由于该列表是从pandas中读出的数据，故不能用if salary_list[i] == np.nan，会识别不出来
            salarys.append(round(salary_list[i]/1000, 1))
    mean = np.mean(salarys)

    plt.figure(dpi=200)
    sns.distplot(salarys, hist = True, kde = True, kde_kws={"color":"r", "lw":1.5, 'linestyle':'-'})
    plt.axvline(mean, color='r', linestyle=":")
    plt.text(mean, 0.01, '平均薪资: %.1f千'%(mean), color='r', horizontalalignment = 'center', fontsize = 15)
    plt.xlim(0,50)
    plt.xlabel('薪资分布（单位：千）')
    plt.title('数据分析职位整体薪资分布')
    plt.savefig('./数据分析职位整体薪资分布.jpg')
    plt.show()


#数据分析职位对学历要求的分布

def education_distribute():

    table = pd.DataFrame()
    for i in citys:
        file_name = './' + i + '.csv'
        df = pd.read_csv(file_name, index_col=0)
        table = pd.concat([table, df])
    table = pd.DataFrame(pd.value_counts(table['education']))
    table = table.sort_values(['education'], ascending = False)
    x = list(table.index)
    y = list(table['education'])
    print(x)

    fig = plt.figure(dpi=200)
    ax = fig.add_axes([0.1,0.1,0.8,0.8])
    explode = (0, 0, 0, 0.2, 0.4, 0.6, 0.8)
    ax.axis('equal')
    ax.pie(y,labels = x,autopct='%.1f%%',explode=explode)   #autopct显示每块饼的百分比属性且自定义格式化字符串，其中%%表示字符串%，类似正则
    ax.set_title('数据分析职位对学历要求的占比')
    ax.legend(x, loc = 1)
    plt.savefig('./数据分析职位对学历要求的占比.jpg')
    plt.show()


#技能关键词频统计

def wordfrequence():

    table = pd.DataFrame()
    for i in citys:
        file_name = './' + i + '.csv'
        df = pd.read_csv(file_name, index_col=0)
        table = pd.concat([table, df])
    l1 = list(table['ability'])
    l2 = list()
    for i in range(len(l1)):
        if not pd.isnull(l1[i]):
            l2.append(l1[i])
    words = ''.join(l2)

    cloud = WordCloud(
        font_path='/System/Library/Fonts/STHeiti Light.ttc',    # 设置字体文件获取路径，默认字体不支持中文
        background_color='white',    # 设置背景颜色  默认是black
        max_words=20,    # 词云显示的最大词语数量
        random_state = 1,  # 设置随机生成状态，即多少种配色方案
        collocations = False,    # 是否包括词语之间的搭配，默认True，可能会产生语意重复的词语
        width=1200, height=900      # 设置大小，默认图片比较小，模糊
    ).generate(words)
    plt.figure(dpi=200)
    plt.imshow(cloud)       # 该方法用来在figure对象上绘制传入图像数据参数的图像
    plt.axis('off')     # 设置词云图中无坐标轴
    plt.savefig("./技能关键词频统计.jpg")
    plt.show()


if __name__ == "__main__":
    
    data_clear()
    citys_jobs()
    citys_salary()
    salary_distribute()
    wordfrequence()

sql复制代码EXPLAIN ANALYZE
 SELECT emp_id, emp_name
   FROM employee e
   WHERE salary > (
     SELECT AVG(salary)
       FROM employee
       WHERE dept_id = e.dept_id);
-> Filter: (e.salary > (select #2))  (cost=2.75 rows=25) (actual time=0.232..4.401 rows=6 loops=1)
    -> Table scan on e  (cost=2.75 rows=25) (actual time=0.099..0.190 rows=25 loops=1)
    -> Select #2 (subquery in condition; dependent)
        -> Aggregate: avg(employee.salary)  (actual time=0.147..0.149 rows=1 loops=25)
            -> Index lookup on employee using idx_emp_dept (dept_id=e.dept_id)  (cost=1.12 rows=5) (actual time=0.068..0.104 rows=7 loops=25)

sql复制代码EXPLAIN ANALYZE
 SELECT e.emp_id, e.emp_name
   FROM employee e
   JOIN (SELECT dept_id, AVG(salary) AS dept_average
           FROM employee
          GROUP BY dept_id) t
     ON e.dept_id = t.dept_id
  WHERE e.salary > t.dept_average;
-> Nested loop inner join  (actual time=0.722..2.354 rows=6 loops=1)
    -> Table scan on e  (cost=2.75 rows=25) (actual time=0.096..0.205 rows=25 loops=1)
    -> Filter: (e.salary > t.dept_average)  (actual time=0.068..0.076 rows=0 loops=25)
        -> Index lookup on t using <auto_key0> (dept_id=e.dept_id)  (actual time=0.011..0.015 rows=1 loops=25)
            -> Materialize  (actual time=0.048..0.057 rows=1 loops=25)
                -> Group aggregate: avg(employee.salary)  (actual time=0.228..0.510 rows=5 loops=1)
                    -> Index scan on employee using idx_emp_dept  (cost=2.75 rows=25) (actual time=0.181..0.348 rows=25 loops=1)

sql复制代码-- MySQL
SELECT *
  FROM large_table
 ORDER BY id
 LIMIT 10 OFFSET N;

sql复制代码-- MySQL
SELECT *
  FROM large_table
 WHERE id > last_id
 ORDER BY id
 LIMIT 10;

sql复制代码(6)SELECT [DISTINCT | ALL] col1, col2, agg_func(col3) AS alias
(1)  FROM t1 JOIN t2
(2)    ON (join_conditions)
(3) WHERE where_conditions
(4) GROUP BY col1, col2
(5)HAVING having_condition
(7) UNION [ALL]
   ...
(8) ORDER BY col1 ASC,col2 DESC
(9)OFFSET m ROWS FETCH NEXT num_rows ROWS ONLY;

sql复制代码-- 错误示例
SELECT emp_name AS empname
  FROM employee
 WHERE empname ='张飞';

vbnet复制代码-- GROUP BY 错误示例
SELECT dept_id, emp_name, AVG(salary)
  FROM employee
 GROUP BY dept_id;

sql复制代码SELECT e.emp_name, d.dept_name
  FROM employee e
  LEFT JOIN department d ON (e.dept_id = d.dept_id)
 WHERE e.emp_name ='张飞';
emp_name|dept_name|
--------|---------|
张飞     |行政管理部|

SELECT e.emp_name, d.dept_name
  FROM employee e
  LEFT JOIN department d ON (e.dept_id = d.dept_id AND e.emp_name ='张飞');
emp_name|dept_name|
--------|---------|
刘备     |   [NULL]|
关羽     |   [NULL]|
张飞     |行政管理部|
诸葛亮   |   [NULL]|
...