MongoDB + ElasticSearch + Nginx 简单试用

install jdk:

wget http://download.oracle.com/otn-pub/java/jdk/8u77-b03/jdk-8u77-linux-x64.rpm #以实际地址为准
sudo rpm -ivh jdk-8u77-linux-x64.rpm
java -version

install elasticsearch:

wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/rpm/elasticsearch/2.3.1/elasticsearch-2.3.1.rpm
sudo rpm -ivh elasticsearch-2.3.1.rpm
sudo chkconfig --add elasticsearch

config elasticsearch:

mkdir -p /home/worker/data/elasticsearch/data
mkdir -p /home/worker/data/elasticsearch/log
mkdir -p /home/worker/data/elasticsearch/pid
sudo chown -R worker:worker /etc/elasticsearch
export ES_HEAP_SIZE=15000000    # half useable memory

sudo vi /etc/sysconfig/elasticsearch

DATA_DIR=/home/worker/data/elasticsearch/data
LOG_DIR=/home/worker/data/elasticsearch/log
PID_DIR=/home/worker/data/elasticsearch/pid
ES_HEAP_SIZE=15g
ES_USER=worker
ES_GROUP=worker

sudo /usr/share/elasticsearch/bin/plugin install lmenezes/elasticsearch-kopf

配置:
sudo vi /etc/elasticsearch/elasticsearch.yml

cluster.name: elasticsearch
node.name: node-${HOSTNAME}
network.host: 10.100.30.105
discovery .zen.ping.unicast.hosts: ["10.100.30.105","10.100.30.106"]

sudo service elasticsearch start

curl 10.100.30.105:9200

mongo-connector elastic2-doc-manager:

sudo pip install elastic2-doc-manager

hosts: 127.0.0.1 search.test.com

nginx负载均衡配置:

upstream elasticsearch {
    server 10.100.30.105:9200;
    server 10.100.30.106:9200;
    keepalive 100;
}
server {
    server_name search.test.com;
    listen 80;
    location / {
        proxy_pass http://elasticsearch;
        proxy_http_version 1.1;
        proxy_set_header Connection "";
        proxy_set_header  X-Real-IP  $remote_addr;
        access_log off;
        allow all;
    }
}

openresty:

wget https://openresty.org/download/openresty-1.9.7.4.tar.gz
./configure --prefix=/home/worker/openresty --with-pcre-jit --with-http_stub_status_module --with-http_ssl_module
sudo /home/worker/nginx/sbin/nginx -s reload

curl search.test.com

ab -n 1000000 -c100 -k http://10.100.30.105:9200/
ab -n 1000000 -c100 -k http://search.test.com/

http://search.test.com/_plugin/kopf/#!/cluster

import MongoDB data:

mongoimport -h 127.0.0.1:27017 -d user_center -c users users.data.201601061

db.users.count()

2640260 264万 0.625GB

config = {
    _id: "RS0",
    members: [
        {_id: 0, host: "10.100.30.52:27017"},
        {_id: 1, host: "10.100.30.52:27018"},
    ]
}

mongo-connector:

config.json
{
    "mainAddress": "10.100.30.52:27017",
    "oplogFile": "/home/worker/data/mongo-connector/oplog.timestamp",
    "noDump": false,
    "batchSize": -1,
    "verbosity": 0,
    "continueOnError": true,
    "logging": {
        "type": "file",
        "filename": "/home/worker/data/mongo-connector/mongo-connector.log"
    },
    "namespaces": {
        "include": ["user_center.users"]
    },
    "fields": ["email", "loginType", "certificated", "destroyed", "regDate", "mobile", "detail"],
    "docManagers": [
        {
            "docManager": "elastic2_doc_manager",
            "targetURL": "10.100.30.106:9200",
            "args": {
                "clientOptions": {"timeout": 200}
            },
            "autoCommitInterval": 0
        }
    ]
}

nohup mongo-connector -c config.json > /home/worker/data/mongo-connector/hup.log 2>&1 &

ab:

query.json
{
    "query": {
        "match": {
            "detail.lowerName": "张三"
        }
    }
}

query1.json
{
    "query": {
        "match": {
            "email": "163.com"
        }
    }
}

ab -n 10000 -c100 -k -p query.json "http://search.test.com/user_center/users/_search"

ab -n 10000 -c100 -k "http://search.test.com/user_center/users/_search?size=1&q=detail.lowerName:张三"

ab -n 10000 -c100 -k "http://search.test.com/user_center/users/_search?size=1&q=email:gmail"

标签: mongodb, elasticsearch, mongo-connector

评论已关闭