1 Kafka and Zookeeper Setup:
Please follow the deployment steps related to Kafka are available at
https://kafka.apache.org/quickstart
https://dzone.com/articles/kafka-clients-at-most-once-at-least-once-exactly-o
A> Zookeeper Setup:
1. Download the latest binary file for zookeeper and install/extract it in Linux/Windows OS
2. Go to ../zookeeper-3.4.10/conf folder and rename the file zoo_sample.config to zoo.config
3. Change the data directory of the path if you want.
4. Go to ../ zookeeper-3.4.10/bin folder and execute the shell script zkServer.sh. Once you run, zookeeper will run on default port 2181
B> Kafka Setup:
1. Download the binary file for kafka and install/extract it in Linux/Windows OS
2. Go to ../kafka_2.10-0.10.2.1/config and edit the server.properties file, change the log directory path if you want
For setting up multi-broker cluster, make a copy the server.properties config file depends on the number of instances you wanted to run.
> cp config/server.properties config/server-1.properties
> cp config/server.properties config/server-2.properties
Now edit these new files and set the following properties:
config/server-1.properties:
broker.id=1
listeners=PLAINTEXT://:9093
log.dir=/tmp/kafka-logs-1
config/server-2.properties:
broker.id=2
listeners=PLAINTEXT://:9094
log.dir=/tmp/kafka-logs-2
3. Go to ../ kafka_2.10-0.10.2.1/bin folder and execute the shell script like below,
$ ./kafka-server-start.sh ../config/server.properties.
4. Once you run, Kafka will run on default port 9092.
C> Create a topic in Kafka
1. Use the below command,
$ ./kafka-topics.sh --create --zookeeper localhost: 2181 --replication-factor 1 --partitions 1 --topic test
For multi-broker cluster, change the number for replication factor option depends on the number of instances you are running.
2. You can see the topics which you created using the below command,
$ ./kafka-topics.sh –list --zookeeper localhost: 2181
cd /opt/ctier/softwares/kafka_2.11-0.10.2.0/bin
zookeeper run:
./zookeeper-server-start.sh ../config/zookeeper.properties &
kafka run:
./kafka-server-start.sh ../config/server.properties &
STREAM Run:
nohup sh standalone.sh &
where url is http://192.168.57.64:8180/STREAM/
For Login mongo db using credential:
mongo --username "kartik" --password "kartik@mandal" --authenticationDatabase "StreamPermissionService" --host 192.168.1.1 --port 27017
Approach 1 :- For replica setup Configuration:
D:\Software\MongoDb\mongodb\bin>
mkdir \data\rs11 \data\rs12 \data\rs13 \data\rs21 \data\rs22 \data\rs23 \data\rs31 \data\rs32 \data\rs33 \data\cfgsvr1 \data\cfgsvr2 \data\cfgsvr3 \data\cfg1 \data\cfg2 \data\cfg3
mongod --configsvr --logpath D:\data\cfgsvr1\1.log --dbpath D:\data\cfgsvr1 --port 27018
mongod --replSet envest --logpath D:\data\rs11\1.log --dbpath D:\data\rs11 --port 28017 --smallfiles --oplogSize 64
mongod --replSet envest --logpath D:\data\rs12\1.log --dbpath D:\data\rs12 --port 28018 --smallfiles --oplogSize 64
mongod --replSet envest --logpath D:\data\rs13\1.log --dbpath D:\data\rs13 --port 28019 --smallfiles --oplogSize 64
mongo --port 28017
rs.initiate()
//enter key button press
rs.add("IN-L1155:28018")
rs.add("IN-L1155:28019")
rs.config()
rs.status()
>use StreamPermissionService
>db.createUser({user:"app",pwd:"app@Strm",roles:[{role:"readWrite",db:"StreamPermissionService"}]});
>db.createCollection("STREAM_COMPANY_PERMISSION_DATA")
>db.createCollection("STREAM_USER_PERMISSION_DATA")
>db.createCollection("STREAM_PERMISSION_DATA_USAGE")
>db.createCollection("STREAM_PERMISSION_DATA_AUDIT")
>db.createCollection("STREAM_COMPANY_STATS")
>db.createCollection("STREAM_COMPANY_STATS_HIST")
>db.createCollection("STREAM_USER_STATS")
>db.createCollection("STREAM_USER_STATS_HIST")
>db.createCollection("STREAM_COMPANY_CONFIG")
>db.createCollection("STREAM_COMPANY_CONFIG_AUDIT")
>db.createCollection("STREAM_ADMIN_AUTH_CONFIG")
>db.createCollection("STREAM_COMPANY_LOGIN_STATS")
>db.createCollection("STREAM_COMPANY_EVENT_STATS")
>db.createCollection("STREAM_COMPANY_EVENT_PERMISSION_DATA")
>db.createCollection("STREAM_COMPANY_NOTIFICATION_CONFIG_DATA")
>show collections
> use admin
switched to db admin
> db.system.users.find()
>use StreamPermissionService
>db.STREAM_COMPANY_EVENT_STATS.find()
Approach 1 :- For Shard setup Configuration:
D:\Software\MongoDb\mongodb\bin>
mkdir \data\rs11 \data\rs12 \data\rs13 \data\rs21 \data\rs22 \data\rs23 \data\rs31 \data\rs32 \data\rs33 \data\cfgsvr1 \data\cfgsvr2 \data\cfgsvr3 \data\cfg1 \data\cfg2 \data\cfg3
mongod --configsvr --logpath D:\data\cfgsvr1\1.log --dbpath D:\data\cfgsvr1 --port 27018
mongod --configsvr --logpath D:\data\cfgsvr2\1.log --dbpath D:\data\cfgsvr2 --port 27019
mongod --configsvr --logpath D:\data\cfgsvr3\1.log --dbpath D:\data\cfgsvr3 --port 27020
mongod --shardsvr --replSet envest --logpath \data\rs11\1.log --dbpath \data\rs11 --port 28017 --smallfiles --oplogSize 64
mongod --shardsvr --replSet envest --logpath \data\rs12\1.log --dbpath \data\rs12 --port 28018 --smallfiles --oplogSize 64
mongod --shardsvr --replSet envest --logpath \data\rs13\1.log --dbpath \data\rs13 --port 28019 --smallfiles --oplogSize 64
mongod --shardsvr --replSet net --logpath \data\rs21\1.log --dbpath \data\rs21 --port 28020 --smallfiles --oplogSize 64
mongod --shardsvr --replSet net --logpath \data\rs22\1.log --dbpath \data\rs22 --port 28021 --smallfiles --oplogSize 64
mongod --shardsvr --replSet net --logpath \data\rs23\1.log --dbpath \data\rs23 --port 28022 --smallfiles --oplogSize 64
mongod --shardsvr --replSet aniket --logpath \data\rs31\1.log --dbpath \data\rs31 --port 28023 --smallfiles --oplogSize 64
mongod --shardsvr --replSet aniket --logpath \data\rs32\1.log --dbpath \data\rs32 --port 28024 --smallfiles --oplogSize 64
mongod --shardsvr --replSet aniket --logpath \data\rs33\1.log --dbpath \data\rs33 --port 28025 --smallfiles --oplogSize 64
mongo --port 28017
rs.initiate()
//enter key button press
rs.add("IN-L1155:28018")
rs.add("IN-L1155:28019")
rs.config()
rs.status()
mongo --port 28020
rs.initiate()
//enter key button press
rs.add("IN-L1155:28021")
rs.add("IN-L1155:28022")
rs.config()
rs.status()
mongo --port 28023
rs.initiate()
//enter key button press
rs.add("IN-L1155:28024")
rs.add("IN-L1155:28025")
rs.config()
rs.status()
mongos --configdb IN-L1155:27018,IN-L1155:27019,IN-L1155:27020 --chunkSize 10
mongo
sh.addShard("envest/IN-L1155:28017")
sh.addShard("net/IN-L1155:28020")
sh.addShard("aniket/IN-L1155:28023")
>sh.status()
>show dbs
>use StreamPermissionService
>db.createUser({user:"application",pwd:"application@Strm",roles:[{role:"readWrite",db:"StreamPermissionService"}]});
>db.createCollection("STREAM_COMPANY_PERMISSION_DATA")
>db.createCollection("STREAM_USER_PERMISSION_DATA")
>db.createCollection("STREAM_PERMISSION_DATA_USAGE")
>db.createCollection("STREAM_PERMISSION_DATA_AUDIT")
>db.createCollection("STREAM_COMPANY_STATS")
>db.createCollection("STREAM_COMPANY_STATS_HIST")
>db.createCollection("STREAM_USER_STATS")
>db.createCollection("STREAM_USER_STATS_HIST")
>db.createCollection("STREAM_COMPANY_CONFIG")
>db.createCollection("STREAM_COMPANY_CONFIG_AUDIT")
>db.createCollection("STREAM_ADMIN_AUTH_CONFIG")
>db.createCollection("STREAM_COMPANY_LOGIN_STATS")
>db.createCollection("STREAM_COMPANY_EVENT_STATS")
>db.createCollection("STREAM_COMPANY_EVENT_PERMISSION_DATA")
>db.createCollection("STREAM_COMPANY_NOTIFICATION_CONFIG_DATA")
>show collections
> use admin
switched to db admin
> db.system.users.find()
>use StreamPermissionService
>db.STREAM_COMPANY_EVENT_STATS.find()
I am Kartik Chandra Mandal from Bangalore, India. Working as software engineer in Java technology. Have good knowledge of OWASP and CSRF in security for applications and it's implementation in Java technology.