From 3b33df87fe6329e5bfd2b28daaeb5a55a8ccdd15 Mon Sep 17 00:00:00 2001 From: Felix Steghofer Date: Wed, 15 Nov 2017 13:40:14 +0100 Subject: [PATCH] added redis tools --- src/DoresA/detect_cusum.pyc | Bin 7183 -> 0 bytes src/DoresA/location.pyc | Bin 489 -> 0 bytes src/redis_tools/compile.sh | 24 + src/redis_tools/configs/redis_local.conf | 679 ++++++++++++++++++++ src/redis_tools/configs/redis_local_f.conf | 28 + src/redis_tools/configs/redis_local_f2.conf | 28 + src/redis_tools/configs/redis_local_f3.conf | 28 + src/redis_tools/configs/redis_local_l.conf | 28 + src/redis_tools/configs/redis_local_l2.conf | 28 + src/redis_tools/configs/redis_local_l3.conf | 28 + src/redis_tools/configs/redis_local_r.conf | 28 + src/redis_tools/configs/redis_local_v.conf | 28 + src/redis_tools/flush-all.sh | 8 + src/redis_tools/list-all-keys.sh | 3 + src/redis_tools/persist-redis.sh | 3 + src/redis_tools/r-4-felix | Bin 0 -> 17928 bytes src/redis_tools/r-4-felix.c | 445 +++++++++++++ src/redis_tools/read-parallel.sh | 4 + 18 files changed, 1390 insertions(+) delete mode 100644 src/DoresA/detect_cusum.pyc delete mode 100644 src/DoresA/location.pyc create mode 100755 src/redis_tools/compile.sh create mode 100755 src/redis_tools/configs/redis_local.conf create mode 100755 src/redis_tools/configs/redis_local_f.conf create mode 100755 src/redis_tools/configs/redis_local_f2.conf create mode 100755 src/redis_tools/configs/redis_local_f3.conf create mode 100755 src/redis_tools/configs/redis_local_l.conf create mode 100755 src/redis_tools/configs/redis_local_l2.conf create mode 100755 src/redis_tools/configs/redis_local_l3.conf create mode 100755 src/redis_tools/configs/redis_local_r.conf create mode 100755 src/redis_tools/configs/redis_local_v.conf create mode 100755 src/redis_tools/flush-all.sh create mode 100755 src/redis_tools/list-all-keys.sh create mode 100755 src/redis_tools/persist-redis.sh create mode 100755 src/redis_tools/r-4-felix create mode 100755 src/redis_tools/r-4-felix.c create mode 100755 src/redis_tools/read-parallel.sh diff --git a/src/DoresA/detect_cusum.pyc b/src/DoresA/detect_cusum.pyc deleted file mode 100644 index a3852f7475732ad80fa0810cc28db22eaeb564c3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7183 zcmb_hTW=f36`mm_S)wJ&mKB?_lX#NUF_qAyWG6xDI(8k)2^_?RK|4t6OQ|JyNUpTp zUGEGfF`>{0yDxny&_1<4qVIhwP#{nJ6D^P*(4y#b`kgaNijptAg(7EnZs$H{&N<5e zW3C{6^XH#7MfRD(^T$dGQ^$)h#C^0<_&E_u(ankbInm9F`*|U0G%vylaeq?yXs3if zDefN<{*<^sEquI83q2!rftL9*;wfflg&@LH?8s-X)m#=+3}*bHY$je?^&-Be1}gd2UvERC=a#* z6U31{f=-eKk*&9SQPX1Rwkc!mmX|hTqt<#}YIRw4tWC_NYPAily+)-OcdNec#4!f zL_7hz)5DRO{ae$o@U?9<4ncGOD(&j<#AXNjRZ-!L^8=De}7?}`fill0Y0Zc524JZR7uI2%q zs6q>>7D)2*8F$YPL&{D#;C_Jx%c~2Fi)m-WoDnd}IUaIP?db$@_M0FAF0*l5frbRgn{k( zTJ1q;#VL9^^M>RMskevEPlJ~IPjRj6;d1-HX%_%JtCCc2>Bt)8Mg0TFyoS+HQ>(a@ zO;ng-9r;1j2I@u}htKEj6E8G+59htKr&VcnnS8xAHt2d*tL=_FoC14m^I_)bcrmfZb=-{q9wyt;L z*kGKG+d3IOOhR?Mur(w>?<5F%ARK4^Fr-v@{swT5c9~wYc_nx1zzWz2tbRDE2IG%gJ5k38ftnGV{X9b z5anLa#yC7d(+k5s4rdBW3%$M~H|1;&4vthCH(+zo&~R=Na073%f02oM?T#%g3juQc z*G#pyCoabb?bX}Zx{o{$%x`-U1UK3qL@(G*9z+vS9%4^8Gdq-4=5k1AhD2$>B;KOf z;~BwI)mq zMgxJ2ydHJ3`z*tjnFa}~MbZL|Lvid~(+Ae#3}hoVwvg7|LOLFv8VZ4FdWI3FTu8Y| zi4t^6IENy^HAUQKRApFR4v*EisS|sU^8orbisBa*H@Os8M48%P-B~-KX3pW`!Vvxn zXAgZ4)acg5e>A&nnkf?2q7 z<%(*haW`{+^=6Nf;GmntxJKMDbz`H|JMi&QQc1y3+~r49x`;cd`u^UM2dgV9mvA#Z zQt$F&*grDn(n?vqhi?_LE)XK@)`Q7=h$>*uWwK1bA4go;DXR-*Rarqp$HleFjxhr8 zB7$B(&IhCMgIGY@YUTZxV7c%P7&k#w;+G_#?jCCwiOALE|Bka&+H?RX$3=&b({q&Z z!F!f!0KV-fn$vzR0Nm!HgPp((1)j+DkP~qrS7+bBXsgGXnZ~QJ$6O}Ey)w|6#kCN zMgA$dAWQP}#JpUUujfw7LT&*g)!YwdLC)oi*mDwpr*g04rerDCB;_vPN3|#}5f$@7 z^p6h)XHusak1l55yr$y>hBE7+7H&?On zSCDygmEkbggl4G-gp?RMLF&AL6d4Fm3Yeu#9mNBxc5s0Ni9M>v+6giT=XxtkVw6J_g;79{g{ zpTv$=c*hAwgZ3m>Er6`Z$d<5gl4(Ox;*{`D3;$K&zb5?GY26c%=L!H|g4#0~(~FLG zUx`JmDWW#P06^Cp!hcivZ$T`Qhrrek#)lxeBVvQ=4v^f^#!+A(p!-LU(P$3!Gn9q~ z_|8*(B>h&p_abUhK#Q6Yv5Kk*csVOjGg>TSLH2rbNEGDtw(!qoWe(B@)%!#%FHW2! z13*n9Qyt1VB^Jptt8Q-s3Vnws-W^T+8({$EL1AEMfh8y*v@(&i$RuKXk$V1l;h%V+ z+^`CaRD@p@BdTAdFEZg*NP?2y026Pp@4KZ{oD%aPnmcejh>8ptrUJfXd5`!fU3s<`Cmix zjq~iX!fkdxSJ>-xTTSk@T@NmVR(4H5=bRlP5a^KKI2Gj8m)Swy;Iar0#N495MVs|@ zz)q8zYm{Q~a2Pk);WbbX;VYs=1t>2}Iv$FB^!dwG_Q%N5k{B0(X@9-=`%nM+)zd$= zmmmJ>Pya0a?W!SO)~K@a7FWhwD2g+qN;#RWJA>LvkleAGI@PqT0s<8?aT$LX0yTlIt11yV01UrZW|z$V6*miIw3M` z4hC$1e7cT=JnmV(qf2xv)kqYzA~j3YoTBD5HLp{1h8jf;9jF=!nsFzgt&xJbuuQZsL+W zBykJlX0G7vjC>O-j?0VkRjK5R%*!HHE%9iPXJ+Iy<_faJ=Y!;5ulsScUT00!@oywl z>U6zcBYZUqiP$RW1ooY}NQLz}Y9XCC#WpgBdL3mT`hV1@9|lb-9{>&+VU6Oe+6`(b o2&oZQCH%&k|2w>4OS#;Q{a&cA5Z#72oXVY$vj|i0moNVOU%K!hcmMzZ diff --git a/src/DoresA/location.pyc b/src/DoresA/location.pyc deleted file mode 100644 index 201a7647931cfdb52a3c37a64cedee21f6c04206..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 489 zcmZut%TB{E5F95hG^M)G8|yWfZ1&$q9q7|n8c-q7?pA>kItfE;)RMB?C`o`77MRjSBiux4Nr-1(SffDZ@2~*!z9h-e(TrLY!txTJK5#>r7JC!PYm&b$#W7}6M zN>pWRaHdS1UlY^CUn~nJwr0cC%gp$^q6U3HFDv87q-7B-R_2s 1000 bytes +# 1kb => 1024 bytes +# 1m => 1000000 bytes +# 1mb => 1024*1024 bytes +# 1g => 1000000000 bytes +# 1gb => 1024*1024*1024 bytes +# +# units are case insensitive so 1GB 1Gb 1gB are all the same. + +################################## INCLUDES ################################### + +# Include one or more other config files here. This is useful if you +# have a standard template that goes to all Redis server but also need +# to customize a few per-server settings. Include files can include +# other files, so use this wisely. +# +# Notice option "include" won't be rewritten by command "CONFIG REWRITE" +# from admin or Redis Sentinel. Since Redis always uses the last processed +# line as value of a configuration directive, you'd better put includes +# at the beginning of this file to avoid overwriting config change at runtime. +# +# If instead you are interested in using includes to override configuration +# options, it is better to use include as the last line. +# +# include /path/to/local.conf +# include /path/to/other.conf + +################################ GENERAL ##################################### + +# By default Redis does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. +daemonize yes + +# Close the connection after a client is idle for N seconds (0 to disable) +timeout 0 + +# TCP keepalive. +# +# If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence +# of communication. This is useful for two reasons: +# +# 1) Detect dead peers. +# 2) Take the connection alive from the point of view of network +# equipment in the middle. +# +# On Linux, the specified value (in seconds) is the period used to send ACKs. +# Note that to close the connection the double of the time is needed. +# On other kernels the period depends on the kernel configuration. +# +# A reasonable value for this option is 60 seconds. +tcp-keepalive 0 + +# Specify the server verbosity level. +# This can be one of: +# debug (a lot of information, useful for development/testing) +# verbose (many rarely useful info, but not a mess like the debug level) +# notice (moderately verbose, what you want in production probably) +# warning (only very important / critical messages are logged) +loglevel notice + +# To enable logging to the system logger, just set 'syslog-enabled' to yes, +# and optionally update the other syslog parameters to suit your needs. +# syslog-enabled no + +# Specify the syslog identity. +# syslog-ident redis + +# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. +# syslog-facility local0 + +# Set the number of databases. The default database is DB 0, you can select +# a different one on a per-connection basis using SELECT where +# dbid is a number between 0 and 'databases'-1 +databases 16 + +################################ SNAPSHOTTING ################################ +# +# Save the DB on disk: +# +# save +# +# Will save the DB if both the given number of seconds and the given +# number of write operations against the DB occurred. +# +# In the example below the behaviour will be to save: +# after 900 sec (15 min) if at least 1 key changed +# after 300 sec (5 min) if at least 10 keys changed +# after 60 sec if at least 10000 keys changed +# +# Note: you can disable saving at all commenting all the "save" lines. +# +# It is also possible to remove all the previously configured save +# points by adding a save directive with a single empty string argument +# like in the following example: +# +# save "" +# "900 1 300 10 60 10000" +# + +# By default Redis will stop accepting writes if RDB snapshots are enabled +# (at least one save point) and the latest background save failed. +# This will make the user aware (in a hard way) that data is not persisting +# on disk properly, otherwise chances are that no one will notice and some +# disaster will happen. +# +# If the background saving process will start working again Redis will +# automatically allow writes again. +# +# However if you have setup your proper monitoring of the Redis server +# and persistence, you may want to disable this feature so that Redis will +# continue to work as usual even if there are problems with disk, +# permissions, and so forth. +stop-writes-on-bgsave-error yes + +# Compress string objects using LZF when dump .rdb databases? +# For default that's set to 'yes' as it's almost always a win. +# If you want to save some CPU in the saving child set it to 'no' but +# the dataset will likely be bigger if you have compressible values or keys. +rdbcompression yes + +# Since version 5 of RDB a CRC64 checksum is placed at the end of the file. +# This makes the format more resistant to corruption but there is a performance +# hit to pay (around 10%) when saving and loading RDB files, so you can disable it +# for maximum performances. +# +# RDB files created with checksum disabled have a checksum of zero that will +# tell the loading code to skip the check. +rdbchecksum yes + +# The working directory. +# +# The DB will be written inside this directory, with the filename specified +# above using the 'dbfilename' configuration directive. +# +# The Append Only File will also be created inside this directory. +# +# Note that you must specify a directory here, not a file name. +#dir /home/tek +dir /home/felix/redis/redis + +################################# REPLICATION ################################# + +# Master-Slave replication. Use slaveof to make a Redis instance a copy of +# another Redis server. Note that the configuration is local to the slave +# so for example it is possible to configure the slave to save the DB with a +# different interval, or to listen to another port, and so on. +# +# slaveof + +# If the master is password protected (using the "requirepass" configuration +# directive below) it is possible to tell the slave to authenticate before +# starting the replication synchronization process, otherwise the master will +# refuse the slave request. +# +# masterauth + +# When a slave loses its connection with the master, or when the replication +# is still in progress, the slave can act in two different ways: +# +# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will +# still reply to client requests, possibly with out of date data, or the +# data set may just be empty if this is the first synchronization. +# +# 2) if slave-serve-stale-data is set to 'no' the slave will reply with +# an error "SYNC with master in progress" to all the kind of commands +# but to INFO and SLAVEOF. +# +slave-serve-stale-data yes + +# You can configure a slave instance to accept writes or not. Writing against +# a slave instance may be useful to store some ephemeral data (because data +# written on a slave will be easily deleted after resync with the master) but +# may also cause problems if clients are writing to it because of a +# misconfiguration. +# +# Since Redis 2.6 by default slaves are read-only. +# +# Note: read only slaves are not designed to be exposed to untrusted clients +# on the internet. It's just a protection layer against misuse of the instance. +# Still a read only slave exports by default all the administrative commands +# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve +# security of read only slaves using 'rename-command' to shadow all the +# administrative / dangerous commands. +slave-read-only yes + +# Slaves send PINGs to server in a predefined interval. It's possible to change +# this interval with the repl_ping_slave_period option. The default value is 10 +# seconds. +# +# repl-ping-slave-period 10 + +# The following option sets the replication timeout for: +# +# 1) Bulk transfer I/O during SYNC, from the point of view of slave. +# 2) Master timeout from the point of view of slaves (data, pings). +# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings). +# +# It is important to make sure that this value is greater than the value +# specified for repl-ping-slave-period otherwise a timeout will be detected +# every time there is low traffic between the master and the slave. +# +# repl-timeout 60 + +# Disable TCP_NODELAY on the slave socket after SYNC? +# +# If you select "yes" Redis will use a smaller number of TCP packets and +# less bandwidth to send data to slaves. But this can add a delay for +# the data to appear on the slave side, up to 40 milliseconds with +# Linux kernels using a default configuration. +# +# If you select "no" the delay for data to appear on the slave side will +# be reduced but more bandwidth will be used for replication. +# +# By default we optimize for low latency, but in very high traffic conditions +# or when the master and slaves are many hops away, turning this to "yes" may +# be a good idea. +repl-disable-tcp-nodelay no + +# Set the replication backlog size. The backlog is a buffer that accumulates +# slave data when slaves are disconnected for some time, so that when a slave +# wants to reconnect again, often a full resync is not needed, but a partial +# resync is enough, just passing the portion of data the slave missed while +# disconnected. +# +# The biggest the replication backlog, the longer the time the slave can be +# disconnected and later be able to perform a partial resynchronization. +# +# The backlog is only allocated once there is at least a slave connected. +# +# repl-backlog-size 1mb + +# After a master has no longer connected slaves for some time, the backlog +# will be freed. The following option configures the amount of seconds that +# need to elapse, starting from the time the last slave disconnected, for +# the backlog buffer to be freed. +# +# A value of 0 means to never release the backlog. +# +# repl-backlog-ttl 3600 + +# The slave priority is an integer number published by Redis in the INFO output. +# It is used by Redis Sentinel in order to select a slave to promote into a +# master if the master is no longer working correctly. +# +# A slave with a low priority number is considered better for promotion, so +# for instance if there are three slaves with priority 10, 100, 25 Sentinel will +# pick the one with priority 10, that is the lowest. +# +# However a special priority of 0 marks the slave as not able to perform the +# role of master, so a slave with priority of 0 will never be selected by +# Redis Sentinel for promotion. +# +# By default the priority is 100. +slave-priority 100 + +# It is possible for a master to stop accepting writes if there are less than +# N slaves connected, having a lag less or equal than M seconds. +# +# The N slaves need to be in "online" state. +# +# The lag in seconds, that must be <= the specified value, is calculated from +# the last ping received from the slave, that is usually sent every second. +# +# This option does not GUARANTEES that N replicas will accept the write, but +# will limit the window of exposure for lost writes in case not enough slaves +# are available, to the specified number of seconds. +# +# For example to require at least 3 slaves with a lag <= 10 seconds use: +# +# min-slaves-to-write 3 +# min-slaves-max-lag 10 +# +# Setting one or the other to 0 disables the feature. +# +# By default min-slaves-to-write is set to 0 (feature disabled) and +# min-slaves-max-lag is set to 10. + +################################## SECURITY ################################### + +# Require clients to issue AUTH before processing any other +# commands. This might be useful in environments in which you do not trust +# others with access to the host running redis-server. +# +# This should stay commented out for backward compatibility and because most +# people do not need auth (e.g. they run their own servers). +# +# Warning: since Redis is pretty fast an outside user can try up to +# 150k passwords per second against a good box. This means that you should +# use a very strong password otherwise it will be very easy to break. +# +# requirepass foobared + +# Command renaming. +# +# It is possible to change the name of dangerous commands in a shared +# environment. For instance the CONFIG command may be renamed into something +# hard to guess so that it will still be available for internal-use tools +# but not available for general clients. +# +# Example: +# +# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 +# +# It is also possible to completely kill a command by renaming it into +# an empty string: +# +# rename-command CONFIG "" +# +# Please note that changing the name of commands that are logged into the +# AOF file or transmitted to slaves may cause problems. + +################################### LIMITS #################################### + +# Set the max number of connected clients at the same time. By default +# this limit is set to 10000 clients, however if the Redis server is not +# able to configure the process file limit to allow for the specified limit +# the max number of allowed clients is set to the current file limit +# minus 32 (as Redis reserves a few file descriptors for internal uses). +# +# Once the limit is reached Redis will close all the new connections sending +# an error 'max number of clients reached'. +# +# maxclients 10000 + +# Don't use more memory than the specified amount of bytes. +# When the memory limit is reached Redis will try to remove keys +# according to the eviction policy selected (see maxmemory-policy). +# +# If Redis can't remove keys according to the policy, or if the policy is +# set to 'noeviction', Redis will start to reply with errors to commands +# that would use more memory, like SET, LPUSH, and so on, and will continue +# to reply to read-only commands like GET. +# +# This option is usually useful when using Redis as an LRU cache, or to set +# a hard memory limit for an instance (using the 'noeviction' policy). +# +# WARNING: If you have slaves attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the slaves are subtracted +# from the used memory count, so that network problems / resyncs will +# not trigger a loop where keys are evicted, and in turn the output +# buffer of slaves is full with DELs of keys evicted triggering the deletion +# of more keys, and so forth until the database is completely emptied. +# +# In short... if you have slaves attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for slave +# output buffers (but this is not needed if the policy is 'noeviction'). +# +# maxmemory +maxmemory 205925520000 + +# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory +# is reached. You can select among five behaviors: +# +# volatile-lru -> remove the key with an expire set using an LRU algorithm +# allkeys-lru -> remove any key accordingly to the LRU algorithm +# volatile-random -> remove a random key with an expire set +# allkeys-random -> remove a random key, any key +# volatile-ttl -> remove the key with the nearest expire time (minor TTL) +# noeviction -> don't expire at all, just return an error on write operations +# +# Note: with any of the above policies, Redis will return an error on write +# operations, when there are not suitable keys for eviction. +# +# At the date of writing this commands are: set setnx setex append +# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd +# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby +# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby +# getset mset msetnx exec sort +# +# The default is: +# +# maxmemory-policy volatile-lru +maxmemory-policy noeviction + +# LRU and minimal TTL algorithms are not precise algorithms but approximated +# algorithms (in order to save memory), so you can select as well the sample +# size to check. For instance for default Redis will check three keys and +# pick the one that was used less recently, you can change the sample size +# using the following configuration directive. +# +# maxmemory-samples 3 + +############################## APPEND ONLY MODE ############################### + +# By default Redis asynchronously dumps the dataset on disk. This mode is +# good enough in many applications, but an issue with the Redis process or +# a power outage may result into a few minutes of writes lost (depending on +# the configured save points). +# +# The Append Only File is an alternative persistence mode that provides +# much better durability. For instance using the default data fsync policy +# (see later in the config file) Redis can lose just one second of writes in a +# dramatic event like a server power outage, or a single write if something +# wrong with the Redis process itself happens, but the operating system is +# still running correctly. +# +# AOF and RDB persistence can be enabled at the same time without problems. +# If the AOF is enabled on startup Redis will load the AOF, that is the file +# with the better durability guarantees. +# +# Please check http://redis.io/topics/persistence for more information. + +appendonly no + +# The name of the append only file (default: "appendonly.aof") + +appendfilename "appendonly.aof" + +# The fsync() call tells the Operating System to actually write data on disk +# instead to wait for more data in the output buffer. Some OS will really flush +# data on disk, some other OS will just try to do it ASAP. +# +# Redis supports three different modes: +# +# no: don't fsync, just let the OS flush the data when it wants. Faster. +# always: fsync after every write to the append only log . Slow, Safest. +# everysec: fsync only one time every second. Compromise. +# +# The default is "everysec", as that's usually the right compromise between +# speed and data safety. It's up to you to understand if you can relax this to +# "no" that will let the operating system flush the output buffer when +# it wants, for better performances (but if you can live with the idea of +# some data loss consider the default persistence mode that's snapshotting), +# or on the contrary, use "always" that's very slow but a bit safer than +# everysec. +# +# More details please check the following article: +# http://antirez.com/post/redis-persistence-demystified.html +# +# If unsure, use "everysec". + +# appendfsync always +appendfsync everysec +# appendfsync no + +# When the AOF fsync policy is set to always or everysec, and a background +# saving process (a background save or AOF log background rewriting) is +# performing a lot of I/O against the disk, in some Linux configurations +# Redis may block too long on the fsync() call. Note that there is no fix for +# this currently, as even performing fsync in a different thread will block +# our synchronous write(2) call. +# +# In order to mitigate this problem it's possible to use the following option +# that will prevent fsync() from being called in the main process while a +# BGSAVE or BGREWRITEAOF is in progress. +# +# This means that while another child is saving, the durability of Redis is +# the same as "appendfsync none". In practical terms, this means that it is +# possible to lose up to 30 seconds of log in the worst scenario (with the +# default Linux settings). +# +# If you have latency problems turn this to "yes". Otherwise leave it as +# "no" that is the safest pick from the point of view of durability. + +no-appendfsync-on-rewrite no + +# Automatic rewrite of the append only file. +# Redis is able to automatically rewrite the log file implicitly calling +# BGREWRITEAOF when the AOF log size grows by the specified percentage. +# +# This is how it works: Redis remembers the size of the AOF file after the +# latest rewrite (if no rewrite has happened since the restart, the size of +# the AOF at startup is used). +# +# This base size is compared to the current size. If the current size is +# bigger than the specified percentage, the rewrite is triggered. Also +# you need to specify a minimal size for the AOF file to be rewritten, this +# is useful to avoid rewriting the AOF file even if the percentage increase +# is reached but it is still pretty small. +# +# Specify a percentage of zero in order to disable the automatic AOF +# rewrite feature. + +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb + +################################ LUA SCRIPTING ############################### + +# Max execution time of a Lua script in milliseconds. +# +# If the maximum execution time is reached Redis will log that a script is +# still in execution after the maximum allowed time and will start to +# reply to queries with an error. +# +# When a long running script exceed the maximum execution time only the +# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be +# used to stop a script that did not yet called write commands. The second +# is the only way to shut down the server in the case a write commands was +# already issue by the script but the user don't want to wait for the natural +# termination of the script. +# +# Set it to 0 or a negative value for unlimited execution without warnings. +lua-time-limit 5000 + +################################## SLOW LOG ################################### + +# The Redis Slow Log is a system to log queries that exceeded a specified +# execution time. The execution time does not include the I/O operations +# like talking with the client, sending the reply and so forth, +# but just the time needed to actually execute the command (this is the only +# stage of command execution where the thread is blocked and can not serve +# other requests in the meantime). +# +# You can configure the slow log with two parameters: one tells Redis +# what is the execution time, in microseconds, to exceed in order for the +# command to get logged, and the other parameter is the length of the +# slow log. When a new command is logged the oldest one is removed from the +# queue of logged commands. + +# The following time is expressed in microseconds, so 1000000 is equivalent +# to one second. Note that a negative number disables the slow log, while +# a value of zero forces the logging of every command. +slowlog-log-slower-than 10000 + +# There is no limit to this length. Just be aware that it will consume memory. +# You can reclaim memory used by the slow log with SLOWLOG RESET. +slowlog-max-len 128 + +############################# Event notification ############################## + +# Redis can notify Pub/Sub clients about events happening in the key space. +# This feature is documented at http://redis.io/topics/keyspace-events +# +# For instance if keyspace events notification is enabled, and a client +# performs a DEL operation on key "foo" stored in the Database 0, two +# messages will be published via Pub/Sub: +# +# PUBLISH __keyspace@0__:foo del +# PUBLISH __keyevent@0__:del foo +# +# It is possible to select the events that Redis will notify among a set +# of classes. Every class is identified by a single character: +# +# K Keyspace events, published with __keyspace@__ prefix. +# E Keyevent events, published with __keyevent@__ prefix. +# g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... +# $ String commands +# l List commands +# s Set commands +# h Hash commands +# z Sorted set commands +# x Expired events (events generated every time a key expires) +# e Evicted events (events generated when a key is evicted for maxmemory) +# A Alias for g$lshzxe, so that the "AKE" string means all the events. +# +# The "notify-keyspace-events" takes as argument a string that is composed +# by zero or multiple characters. The empty string means that notifications +# are disabled at all. +# +# Example: to enable list and generic events, from the point of view of the +# event name, use: +# +# notify-keyspace-events Elg +# +# Example 2: to get the stream of the expired keys subscribing to channel +# name __keyevent@0__:expired use: +# +# notify-keyspace-events Ex +# +# By default all notifications are disabled because most users don't need +# this feature and the feature has some overhead. Note that if you don't +# specify at least one of K or E, no events will be delivered. +notify-keyspace-events "" + +############################### ADVANCED CONFIG ############################### + +# Hashes are encoded using a memory efficient data structure when they have a +# small number of entries, and the biggest entry does not exceed a given +# threshold. These thresholds can be configured using the following directives. +# values determined by empirical measuring.. +hash-max-ziplist-entries 15000 +#hash-max-ziplist-value 128 +hash-max-ziplist-value 400 + +# Similarly to hashes, small lists are also encoded in a special way in order +# to save a lot of space. The special representation is only used when +# you are under the following limits: +list-max-ziplist-entries 512 +list-max-ziplist-value 64 + +# Sets have a special encoding in just one case: when a set is composed +# of just strings that happens to be integers in radix 10 in the range +# of 64 bit signed integers. +# The following configuration setting sets the limit in the size of the +# set in order to use this special memory saving encoding. +set-max-intset-entries 512 + +# Similarly to hashes and lists, sorted sets are also specially encoded in +# order to save a lot of space. This encoding is only used when the length and +# elements of a sorted set are below the following limits: +zset-max-ziplist-entries 128 +zset-max-ziplist-value 64 + +# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in +# order to help rehashing the main Redis hash table (the one mapping top-level +# keys to values). The hash table implementation Redis uses (see dict.c) +# performs a lazy rehashing: the more operation you run into a hash table +# that is rehashing, the more rehashing "steps" are performed, so if the +# server is idle the rehashing is never complete and some more memory is used +# by the hash table. +# +# The default is to use this millisecond 10 times every second in order to +# active rehashing the main dictionaries, freeing memory when possible. +# +# If unsure: +# use "activerehashing no" if you have hard latency requirements and it is +# not a good thing in your environment that Redis can reply form time to time +# to queries with 2 milliseconds delay. +# +# use "activerehashing yes" if you don't have such hard requirements but +# want to free memory asap when possible. +activerehashing yes + +# The client output buffer limits can be used to force disconnection of clients +# that are not reading data from the server fast enough for some reason (a +# common reason is that a Pub/Sub client can't consume messages as fast as the +# publisher can produce them). +# +# The limit can be set differently for the three different classes of clients: +# +# normal -> normal clients +# slave -> slave clients and MONITOR clients +# pubsub -> clients subscribed to at least one pubsub channel or pattern +# +# The syntax of every client-output-buffer-limit directive is the following: +# +# client-output-buffer-limit +# +# A client is immediately disconnected once the hard limit is reached, or if +# the soft limit is reached and remains reached for the specified number of +# seconds (continuously). +# So for instance if the hard limit is 32 megabytes and the soft limit is +# 16 megabytes / 10 seconds, the client will get disconnected immediately +# if the size of the output buffers reach 32 megabytes, but will also get +# disconnected if the client reaches 16 megabytes and continuously overcomes +# the limit for 10 seconds. +# +# By default normal clients are not limited because they don't receive data +# without asking (in a push way), but just after a request, so only +# asynchronous clients may create a scenario where data is requested faster +# than it can read. +# +# Instead there is a default limit for pubsub and slave clients, since +# subscribers and slaves receive data in a push fashion. +# +# Both the hard or the soft limit can be disabled by setting them to zero. +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit slave 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 + +# Redis calls an internal function to perform many background tasks, like +# closing connections of clients in timeout, purging expired keys that are +# never requested, and so forth. +# +# Not all tasks are performed with the same frequency, but Redis checks for +# tasks to perform accordingly to the specified "hz" value. +# +# By default "hz" is set to 10. Raising the value will use more CPU when +# Redis is idle, but at the same time will make Redis more responsive when +# there are many keys expiring at the same time, and timeouts may be +# handled with more precision. +# +# The range is between 1 and 500, however a value over 100 is usually not +# a good idea. Most users should use the default of 10 and raise this up to +# 100 only in environments where very low latency is required. +hz 10 + +# When a child rewrites the AOF file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +aof-rewrite-incremental-fsync yes + +protected-mode no diff --git a/src/redis_tools/configs/redis_local_f.conf b/src/redis_tools/configs/redis_local_f.conf new file mode 100755 index 0000000..8e6f478 --- /dev/null +++ b/src/redis_tools/configs/redis_local_f.conf @@ -0,0 +1,28 @@ +# Redis configuration file example +################################## INCLUDES ################################### + +include redis_local.conf + +# When running daemonized, Redis writes a pid file in /var/run/redis.pid by +# default. You can specify a custom pid file location here. +pidfile /home/felix/redis/redis/redis-server_f.pid + +# Accept connections on the specified port, default is 6379. +# If port 0 is specified Redis will not listen on a TCP socket. +port 2337 + +# Specify the path for the unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# yields better performance than loopback +unixsocket /home/felix/redis/redis/redis_local_f.sock +unixsocketperm 755 + +# Specify the log file name. Also the empty string can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile /home/felix/redis/redis/redis-server_f.log + +# The filename where to dump the DB +dbfilename dump_f.rdb + diff --git a/src/redis_tools/configs/redis_local_f2.conf b/src/redis_tools/configs/redis_local_f2.conf new file mode 100755 index 0000000..43bf41e --- /dev/null +++ b/src/redis_tools/configs/redis_local_f2.conf @@ -0,0 +1,28 @@ +# Redis configuration file example +################################## INCLUDES ################################### + +include redis_local.conf + +# When running daemonized, Redis writes a pid file in /var/run/redis.pid by +# default. You can specify a custom pid file location here. +pidfile /home/felix/redis/redis/redis-server_f2.pid + +# Accept connections on the specified port, default is 6379. +# If port 0 is specified Redis will not listen on a TCP socket. +port 2338 + +# Specify the path for the unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# yields better performance than loopback +unixsocket /home/felix/redis/redis/redis_local_f2.sock +unixsocketperm 755 + +# Specify the log file name. Also the empty string can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile /home/felix/redis/redis/redis-server_f2.log + +# The filename where to dump the DB +dbfilename dump_f2.rdb + diff --git a/src/redis_tools/configs/redis_local_f3.conf b/src/redis_tools/configs/redis_local_f3.conf new file mode 100755 index 0000000..6bd7453 --- /dev/null +++ b/src/redis_tools/configs/redis_local_f3.conf @@ -0,0 +1,28 @@ +# Redis configuration file example +################################## INCLUDES ################################### + +include redis_local.conf + +# When running daemonized, Redis writes a pid file in /var/run/redis.pid by +# default. You can specify a custom pid file location here. +pidfile /home/felix/redis/redis/redis-server_f3.pid + +# Accept connections on the specified port, default is 6379. +# If port 0 is specified Redis will not listen on a TCP socket. +port 2339 + +# Specify the path for the unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# yields better performance than loopback +unixsocket /home/felix/redis/redis/redis_local_f3.sock +unixsocketperm 755 + +# Specify the log file name. Also the empty string can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile /home/felix/redis/redis/redis-server_f3.log + +# The filename where to dump the DB +dbfilename dump_f3.rdb + diff --git a/src/redis_tools/configs/redis_local_l.conf b/src/redis_tools/configs/redis_local_l.conf new file mode 100755 index 0000000..b9f8340 --- /dev/null +++ b/src/redis_tools/configs/redis_local_l.conf @@ -0,0 +1,28 @@ +# Redis configuration file example +################################## INCLUDES ################################### + +include redis_local.conf + +# When running daemonized, Redis writes a pid file in /var/run/redis.pid by +# default. You can specify a custom pid file location here. +pidfile /home/felix/redis/redis/redis-server_l.pid + +# Accept connections on the specified port, default is 6379. +# If port 0 is specified Redis will not listen on a TCP socket. +port 2340 + +# Specify the path for the unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# yields better performance than loopback +unixsocket /home/felix/redis/redis/redis_local_l.sock +unixsocketperm 755 + +# Specify the log file name. Also the empty string can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile /home/felix/redis/redis/redis-server_l.log + +# The filename where to dump the DB +dbfilename dump_l.rdb + diff --git a/src/redis_tools/configs/redis_local_l2.conf b/src/redis_tools/configs/redis_local_l2.conf new file mode 100755 index 0000000..dbd6920 --- /dev/null +++ b/src/redis_tools/configs/redis_local_l2.conf @@ -0,0 +1,28 @@ +# Redis configuration file example +################################## INCLUDES ################################### + +include redis_local.conf + +# When running daemonized, Redis writes a pid file in /var/run/redis.pid by +# default. You can specify a custom pid file location here. +pidfile /home/felix/redis/redis/redis-server_l2.pid + +# Accept connections on the specified port, default is 6379. +# If port 0 is specified Redis will not listen on a TCP socket. +port 2341 + +# Specify the path for the unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# yields better performance than loopback +unixsocket /home/felix/redis/redis/redis_local_l2.sock +unixsocketperm 755 + +# Specify the log file name. Also the empty string can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile /home/felix/redis/redis/redis-server_l2.log + +# The filename where to dump the DB +dbfilename dump_l2.rdb + diff --git a/src/redis_tools/configs/redis_local_l3.conf b/src/redis_tools/configs/redis_local_l3.conf new file mode 100755 index 0000000..ff5ebde --- /dev/null +++ b/src/redis_tools/configs/redis_local_l3.conf @@ -0,0 +1,28 @@ +# Redis configuration file example +################################## INCLUDES ################################### + +include redis_local.conf + +# When running daemonized, Redis writes a pid file in /var/run/redis.pid by +# default. You can specify a custom pid file location here. +pidfile /home/felix/redis/redis/redis-server_l3.pid + +# Accept connections on the specified port, default is 6379. +# If port 0 is specified Redis will not listen on a TCP socket. +port 2342 + +# Specify the path for the unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# yields better performance than loopback +unixsocket /home/felix/redis/redis/redis_local_l3.sock +unixsocketperm 755 + +# Specify the log file name. Also the empty string can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile /home/felix/redis/redis/redis-server_l3.log + +# The filename where to dump the DB +dbfilename dump_l3.rdb + diff --git a/src/redis_tools/configs/redis_local_r.conf b/src/redis_tools/configs/redis_local_r.conf new file mode 100755 index 0000000..15f425a --- /dev/null +++ b/src/redis_tools/configs/redis_local_r.conf @@ -0,0 +1,28 @@ +# Redis configuration file example +################################## INCLUDES ################################### + +include redis_local.conf + +# When running daemonized, Redis writes a pid file in /var/run/redis.pid by +# default. You can specify a custom pid file location here. +pidfile /home/felix/redis/redis/redis-server_r.pid + +# Accept connections on the specified port, default is 6379. +# If port 0 is specified Redis will not listen on a TCP socket. +port 2343 + +# Specify the path for the unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# yields better performance than loopback +unixsocket /home/felix/redis/redis/redis_local_r.sock +unixsocketperm 755 + +# Specify the log file name. Also the empty string can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile /home/felix/redis/redis/redis-server_r.log + +# The filename where to dump the DB +dbfilename dump_r.rdb + diff --git a/src/redis_tools/configs/redis_local_v.conf b/src/redis_tools/configs/redis_local_v.conf new file mode 100755 index 0000000..8f34804 --- /dev/null +++ b/src/redis_tools/configs/redis_local_v.conf @@ -0,0 +1,28 @@ +# Redis configuration file example +################################## INCLUDES ################################### + +include redis_local.conf + +# When running daemonized, Redis writes a pid file in /var/run/redis.pid by +# default. You can specify a custom pid file location here. +pidfile /home/felix/redis/redis/redis-server_v.pid + +# Accept connections on the specified port, default is 6379. +# If port 0 is specified Redis will not listen on a TCP socket. +port 2344 + +# Specify the path for the unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# yields better performance than loopback +unixsocket /home/felix/redis/redis/redis_local_v.sock +unixsocketperm 755 + +# Specify the log file name. Also the empty string can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile /home/felix/redis/redis/redis-server_v.log + +# The filename where to dump the DB +dbfilename dump_v.rdb + diff --git a/src/redis_tools/flush-all.sh b/src/redis_tools/flush-all.sh new file mode 100755 index 0000000..693e433 --- /dev/null +++ b/src/redis_tools/flush-all.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +read -p "Really delete all redis data? " -n 1 -r +echo +if [[ $REPLY =~ ^[Yy]$ ]] +then + for i in `seq 2337 2344`; do redis-cli -p "$i" flushall; done +fi diff --git a/src/redis_tools/list-all-keys.sh b/src/redis_tools/list-all-keys.sh new file mode 100755 index 0000000..7abbd7a --- /dev/null +++ b/src/redis_tools/list-all-keys.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +for i in `seq 2337 2344`; do redis-cli -p "$i" keys \*; done diff --git a/src/redis_tools/persist-redis.sh b/src/redis_tools/persist-redis.sh new file mode 100755 index 0000000..127b6e5 --- /dev/null +++ b/src/redis_tools/persist-redis.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +for i in `seq 2337 2344`; do redis-cli -p "$i" bgsave & sleep 4m; done diff --git a/src/redis_tools/r-4-felix b/src/redis_tools/r-4-felix new file mode 100755 index 0000000000000000000000000000000000000000..6b214facb1c9ea0b28882242964cdfa658e92554 GIT binary patch literal 17928 zcmeHPeRP!7nZJ`sARze=5hCg`MPs2t2%iR9Y$h=9rYC$ffQ4;wOfnNPb&{D*W^C{j zO?McZ=``I`p>@06mebSiTD#j-dqlJcoJdP(k8Q*D!}P;8qQZ>9CDyM<(b?Z~KW6Sb znU~7`vuDroYUa7m@A!duvpO~n zzt^&B*c9+3924?-K_C@nLb_J8RN{8fV*J$=2_Tq4>?l048?lz^F{%I@Q^qd4Ddsg#P%vOTMDsg?07D!a>;_{sW?0T(QHz|qVC zk~6>x;S`^vfc}px^ygamKWCx8&_e$T3;cBpKOeEsKW3r7(?Y+@0&lj!YvJF4zx*=a zLVq_D3ypJ$g?_?9zt%$kSquFG7WmIB@L>zQ#{$2}BA%x$^gjyRfxrB+6F{NySquO3 z^OeB|fmgCQY`oTCl14dR#U}Fb&q_RB|6gG}lU>FhzRcu?eV^myY#$wHdckqu;R|^^ zzHU0$YipJ+XYR&r8{Hmn$h+Iu9rlK{ZCvXQba}V6wEDfw?cUuP=yG?5TS8&CTc{RP z1rq4 zE>aFg!rc%e{I)yXZ5?;G+gp5o77q9WpY?`V`))79(DlPpI1oX2A=c^bB+K?Tf1umT zI$QkyKpSfb2Yft$4r;5dr-g!R@%!%baV_)!E|f;=N*Sucsy40%>b^w-5qC z=QlJpHmr3otGOv#Te>1!&DECGtfa{+;^$ua75~|@H99BLHH2spzq00Z0RgjR|05nJ zVfF~9x_->@`EaLD!uoiAF}D9JU&UppnB6D&SzK;s|04OwT;DYE`fuQ=sOzt~ZU*HU zd_-Pn6<#axi6sthr*M_GDVgsISNyQlr)wi!Q&btz@EHmO9o6s(4Nqyfx+fv=xQ73o zre7uFnF>sHl}jBOPGhdh0u5K!bduF-xPISLr{Vf_zh1+sZB+~nS7Su7EgFuA$(9`& zK0S*vcDsh_r;H8_m$#1GDyZS*nx7sGSGh#uUJbuo)9=%8{X99K;a6z-4`?`D%T+n3 z;a4gU^g#`ut>F)A_#6#Ctl?D}p49NGH2j2y&(-i@4Zm8$M>PCe4L71^ON~TH_4N*B z#FF8n%&-ytaVcHxMaisO22rMZ1%4~$)uTpy2MN=o8IB1`QH;ysgDA5%K3+gznu7C&VQSDN~v^`^Is>PQYrl~=kFt)QYd|p^Is&MQYSsY`F|juQYPKY z`MZdxR7nRpA0nPoBz-&hrLQ!`Pu^z4PZ`k{$G2>2>MuDDUq*jfl#?~5p_sYymiZ3$ zdd0kTBw5o42iuMQ5+|s!dcugm6ut^6FcUhI0%f~0qwO^n^Y+1#v0cY#)^j}Ur;%O% z$|CM&;k&Rm2H&+C@o{4?^-;ri+&KAexC#z`_W|K>2066y*SqEkw96t(jOd#GgaIN$ zJY_~=&DVivuKWhD)CP?E@sa@mwq3{I3jKIZQc~ZQZifQZwi=&5J5kSA{K$-_G8rTO z7z6PbiQ?)7j4>nbN-a$e#jXbkmKj5_I$4c!1KT|dj6{rVj0Cigqq9H8OdTfW)P3({ zGKYDd|2<}3=|z(`d}ug+07WA{l)4O)V#H(hP5u1Wz5FMLM&74LvVr<{Vk!SSBMxk?*4q_V8QVJrLtk{>tVJwAD6hZjpeEb|; zB-MgA2r*!NFhwgwSjD~pGD-Av5N^+9GEZi3FdOk-kmpkPSm` z(yOGjkxRpKbCO~yS;8fe8OLLzzeJ4b7m$uQty4J3koRdq%S#e;6p$N$qslqLXF3&;6R&&e}Zon zI~{Lkw_hye_7KkhTI_!|$G(Fc=4N>{Sj$<4shH*0pkkNo~z}sh*Q(s$aisSdJ9V83|4e^bM9PI zDb7r<^5bGC^(wSt$;eXS&gg%PlE~=q7s+{G6dh@dkMncW@vQ4_z-TCT4SGg1OzlIB z;@k>>5kE%AIe|C{v5gqfBrXQA0hk~ZPNGR0nIySM>;aOPnuVSKSB6EbqFYQR0w0{A z)bnq_E}ATY^Mk@UC(?PFaQ+~7et>r^5fj|X8+5M^kz%eN3GyoXA=}wQ2=51?jy)`z z;zTwz!A)X^Wk2{R^5H{21i11EeuNE)e$>epX`K33en*H>t?+`AKUscF!y+X%M<)E{%K5Oe-+@5i%^^5CozYQ^3#JHzgMW6(H~bH=aKkjMCAX4 z*Dvb-cDOCk=tR}NR{5mR@unpnQ6SZ|DdJ+6Lbfp2I)0; z5uK=|P9*y2F7?Tsa~?k#Kk?S7`0!w=%$KbEAY%&$<0t66u@84Z+Tp$jr-(Y3;2hLL zL|jdN4zJOP;s{PFYj&f#)aU-fRk$XyY9qCpHKe0R8t&H&Yshe|*>D!g^?RPx{1~hF zLL_543u$2)Iq$19@58u5OTF}}DXc@(ST-!mZ5h@(5UBoqjSTIE4Nrs33yV_qVqsBQO7Eqb_iy6{D)sm{ zWC{N8LZ^m4YAici!JOeQsaZMuJ~F)9Y`A%HShZSM>N#nMpDb`tA>Sx$Q6o*Nsf#xP z`7>SN65^^0c37~7c|b#{L9k<242)F{jFkNV z=1VD9%Jj3ymcNF7uF?Y`eS4*Gh;J(_n5t} zG=-&Q`sXym*<^U5+3*6`ys#GijnUFF{qHpIhq309`uQuS(fTGemK`lE(_f-yWqxkNaK0ib)+xJtEj1sr+rh_Dc4Cu>5jq zBzC1d#`0_Ul0~>Ab^a_3!LEs)9-|cwtk*e=_}#lM#7ac zY{M0YSjC}a+%=JKO(3jKLKDo8>vb|2yDs6nu-r9)=ZOl`Y{`|6UprVFcU>5>pY&ch z>79tWUbjW}zCQKtXQQrhTfSpF>)a2sw}#=Ep20;}@x2$guj+?T@Yv+Ka57cB9R+un zE)orn7F9ok!exW0qUz-+Ibm`2IEoY&S686qgjE6`Ev?2{7Mn)i2UB2h`jJePQr@Ko zsTPAYf~T47!+ow_54lp*^C8z+A{e9*ECr1l(Y@m=vVsqK`ny6fj-f*jrwi~aIk3+4 z>qv3bm9iOPP*NXu4M%=4HaoHXEc}g&L5EU!PSkbQWQJVnHl zlr9gCQQBaZ$j8QU_`zZvUzfNQ65(S4o{#gGC&o&6RM|YBE~et!N8=~f#a$!uzw~W? zvCs8K^eV+7Q@QHcZ52m1zL?lPN_inXKgXA+ROhAIO;Gz|V*3d7M5rASY7@L~2({#8 z%KN<&SrCFTZxJK>tK10ZhW%&`tivcz@7;Tb zRqRjpxz12Hm?4@Bj^xVgF;|Fb*kM^v4xI#Q$lN4Bn8=csPQ z|E#*LI>L;tu5Fuk2!uf5b6hYK2nX5%eusD)j!N>gRaAR4_%`E06Gf~roQu;%nHyZtP*YmeHJtfI)Q861E3vP!X5>sf)!F;YKRi@2W(R-38(j~Bd0T&8-)u0Ev(Pj^-{kSF&-q}<(2Ept3Fn7*=I|8*+=Jo z_(RLDr$uMxz#m>iurHzyr?8^KoaL4G7OgFtVnprs-&CS?(QXGs2G)PR2Ab6=x-Tp5 z1N;=^UpC3(SvifeNi#mXW6&{AnL;m z$aY|Cy{NN>^2&cKYACPzvfWkg=r3+4U+|?8qrCRsDMoo+bn3?P`cQeDv%J<>zM!Gp z0X^t9l$Y}N_cTYn@FUJRG8tKzwf^R?0t5BHo`y;IVACH<OwoiXI(OH%=jEOS`e?A^gTA&*_k?v=Dg7Pp_skOSLAc3 zIH?rE>4UT)jPt>~yr%OZ;iGx@0*R~ZfNF!b@z_`{PIaEz$nnYJ7M6b0bw&B<<@%Gy z|69Nd#ZRZlLiqQ&pUc?{3OKJ6(&rzp;EVPA`TZGeZFs88pI=(AqA%Rl^^3FkIn67R z^X+ZmwgU5Bfq|Ndaqg9Qs}ApZ5`S%;@XVhJQT-st1*ZS5w!k+5cj$_;z6lLEaMjAs zw@+L6x!VH2pW|ih^Rf~Ce2O-{VWIy6;Dz=VJZ7O!&#N;F9PdB1(0`8OW$aUO-1zf0 z;(6IZe+tGy|Lzy6ynG8c0EO_m7W%6#@Fw8Y-~4gjX`$cA@iKPq10oLoypHM~;0^(q ze*1u5UNk3r9a6#r7XBY3eeFD{?iU`n&_7{;KWBlT;dmLVl=;lp5)jl|7W$W2jPp-m zgQIBDev@YDU+p8jQFizS?x&1-@7Pv0cR+pMK))oqPha>In zsOEO!xWk=pY(wet(jFF1z`ffaXvO-kCmaZMyIUeXtS!(P^n1fzPtB^;H(?`BAtk1#bUk~jkw(I_07(WF1KsbI@$?j zR`xLWx}BSx8ynY}C4A!$Al$LJU54~vtZQcO4NaRHoK5b{>(^r`)4k2v(BvYod>2hy zcSP*|33>1IhOiBXb_mtit4%YsD@N|{F@qP|$t1Q9nf2wKA2WzeLG3=T-@`W#nMHCp zkJuYz7KvR%W~?pLwtShpEz%thbfRnKT}V0A6>?`$4xtWC+E8Q`^Sw)E2-A=CWx30U zy4x8vE2y4%yneVhZ|rh=x&!VG?90NYFR=wGXN#>#?#9hYrOhkf+aXH0R3O=4z@ z$7bG;Mw`6MN^-B68Fag`Lrxt9v~})1?1wY&88iE1HQjqU!!50#;gFyms)n6z-cXR$ z;9Zfo#@W!gINY*Za=W`CHLVdJcJTT%e}aZXLP7_TFVJOT+>nL5 z{uVNjwV*%DYWV!rguOke^Bs6Kp#a~*R^#oECyNdbY!xRg#92dFs5%_Cbo$!RY5>0Q zQ^Ok&&M=Ft2IsF%oRbQiFE16MbqyQ-aGK>sJ?|;1)-|M<>#91@>+na9_{v^A7b@z2 zOfTwsuCS%3==SRQQPJy_qNMru>wx2f$AD^IJ!dLfC+*bwi(IbAdfkK?J=!UIwH~AB z?YJ&cq4gRXCuOhJMd&3LsVIB(+^VQr&w&ar0O2=f6r^-p=N6!s3 zCKMkPwLYV$TDKBX5v!^bjpI+ZSI_B+9?TnXdId&B(fz3F_UgV((U}yqR+OEBzK)7+ zuhtJ0RqJRkc0?H_`W;#bY6^#(=h z{iR;=+fPDBx2WwkvRCvpB)WZe-9#FWW-UcU*(rK1&t5%; zjY@kJr*fn06#oV)gsJ$|efYSv&)3hJPIex)gh_SOL{avaD?n0ZuTzyfx@sw%*ijD4 t0O&fi2!ARbb^fH|L&c72wj@`Jsy#1ZRl7=&;wxl-Y>l+fGst7v{{TUfYlr{< literal 0 HcmV?d00001 diff --git a/src/redis_tools/r-4-felix.c b/src/redis_tools/r-4-felix.c new file mode 100755 index 0000000..97dedd8 --- /dev/null +++ b/src/redis_tools/r-4-felix.c @@ -0,0 +1,445 @@ +#include +#include +#include +#include +#include +// #include + +#define SOCK_F "/home/felix/redis/redis/redis_local_f.sock" +#define SOCK_F2 "/home/felix/redis/redis/redis_local_f2.sock" +#define SOCK_F3 "/home/felix/redis/redis/redis_local_f3.sock" +#define SOCK_L "/home/felix/redis/redis/redis_local_l.sock" +#define SOCK_L2 "/home/felix/redis/redis/redis_local_l2.sock" +#define SOCK_L3 "/home/felix/redis/redis/redis_local_l3.sock" +#define SOCK_R "/home/felix/redis/redis/redis_local_r.sock" +#define SOCK_V "/home/felix/redis/redis/redis_local_v.sock" + +//#define MOD 524288 +#define MOD 1048576 +#define FLUSHMOD 10000 +#ifdef DEBUG +#define CONT(x) printf("%s\n", x); \ + printf("%s", line); \ + continue +#else +#define CONT(x) continue +#endif + +unsigned int crc32_custom(unsigned char *message); +unsigned reverse(unsigned x); +char *getfield(const char *s, const char del, const unsigned int n); +char *get2fields(const char *s, const char del1, const unsigned int n1, const char del2, const unsigned int n2); +char *getfielddel(const char *s, const char del, const char quote, const unsigned int n); + +int main(int argc, char **argv) { + int i; + unsigned int count = 0; + unsigned int count_fl, count_fl2, count_fl3; + redisContext *c_f; + redisContext *c_f2; + redisContext *c_f3; + redisContext *c_l; + redisContext *c_l2; + redisContext *c_l3; + redisContext *c_r; + redisContext *c_v; + redisReply **reply = 0; + + if (argc < 2) { + printf("usage %s filename\n", argv[0]); + return 1; + } + + char line[4096]; + char *tmp, *tmp2; + char rdata[4096]; + char rrname[4096]; + char rrtype[4096]; + int ts; + + // TODO gz + // gzFile f = gzopen(argv[1], "r") + FILE *f = fopen(argv[1], "r"); + + if (!f) { + printf("file %s could not be opened\n", argv[1]); + return 1; + } + + c_f = redisConnectUnix(SOCK_F); + c_f2 = redisConnectUnix(SOCK_F2); + c_f3 = redisConnectUnix(SOCK_F3); + c_l = redisConnectUnix(SOCK_L); + c_l2 = redisConnectUnix(SOCK_L2); + c_l3 = redisConnectUnix(SOCK_L3); + c_r = redisConnectUnix(SOCK_R); + c_v = redisConnectUnix(SOCK_V); + + if (!c_f || !c_f2 || !c_f3 || !c_l || !c_l2 || !c_l3 || !c_r || !c_v) { + perror("uh oh:"); + return 1; + } + + // TODO gz + // while(gzgets(f, line, 4096)) { + while(fgets(line, 4096, f)) { + + if (!strstr(line, "\n")) { + CONT("no newline in line buffer found. read incomplete"); + } + + if (!(tmp = getfielddel(line, ',', '"', 2))) { + CONT("RRTYPE could not be parsed"); + } + strncpy(rrtype, tmp, 4095); + free(tmp); + + for (i = 0; rrtype[i]; i++) + rrtype[i] = toupper(rrtype[i]); + + if (!strcmp(rrtype, "RRSIG")) { + //CONT("RRSIG skipped"); + continue; + } + + if (!strcmp(rrtype, "TEXT") || !strcmp(rrtype, "SOA")) { + rdata[0] = '\0'; + } else { if (!strcmp(rrtype, "SRV")) { + + if (!(tmp = getfielddel(line, ',', '"', 4))) { + continue; // not parsed correctly (bcause of , withing "") // should be fixed by getfield with quote + CONT("SRV query with too few fields"); + } + strncpy(rdata, tmp, 4095); + free(tmp); + } else { + if (!(tmp = getfielddel(line, ',', '"', 0))) { + CONT("timestamp could not be parsed"); + } + ts = atoi(tmp); + free(tmp); + + if (!(tmp = getfielddel(line, ',', '"', 1))) { + CONT("RRNAME could not be parsed"); + } + strncpy(rrname, tmp, 4095); + free(tmp); + + for (i = 0; rrname[i]; i++) + rrname[i] = tolower(rrname[i]); + + if (!(tmp = getfielddel(line, ',', '"', 3))) { + CONT("RDATA could not be parsed"); + } + strcpy(rdata, tmp); + free(tmp); + } + } + + + if (ts < 0 || ts > 2147483648) + printf("WARNING timestamp malformed: %s", line); + + if (argc > 2 && !strcmp(argv[2], "-v")) + printf("ts %u, rrname %s, rrtype %s, rdata %s\n", ts, rrname, rrtype, rdata); + //continue; // TODO remove XXX + + + unsigned int ip = 0; + unsigned int o[4]; + + unsigned int bucket; + char bucket_c[8]; + + char *pdns_r, *pdns_v, *pdns_fl; + size_t pdns_r_l, pdns_v_l, pdns_fl_l; + + pdns_r_l = strlen("r:") + strlen(rrname) + strlen(":") + strlen(rrtype); + pdns_r = malloc(pdns_r_l + 1); + sprintf(pdns_r, "r:%s:%s", rrname, rrtype); + + pdns_v_l = strlen("v:") + strlen(rdata); + pdns_v = malloc(pdns_v_l + 1); + sprintf(pdns_v, "v:%s", rdata); + + pdns_fl_l = strlen(rrname) + strlen(":") + strlen(rdata); + pdns_fl = malloc(pdns_fl_l + 1); + sprintf(pdns_fl, "%s:%s", rrname, rdata); + + if (!strcmp(rrtype, "A")) { + sscanf(rdata, "%u.%u.%u.%u", &o[0], &o[1], &o[2], &o[3]); + for (i=0; i<4; i++) + ip |= o[i] << (8 * (3-i)); + redisAppendCommand(c_r, "SADD %b %b", pdns_r, pdns_r_l, &ip, (size_t) 4); + } else { + redisAppendCommand(c_r, "SADD %b %b", pdns_r, pdns_r_l, &rdata, strlen(rdata)); + } + + redisAppendCommand(c_v, "SADD %b %b", pdns_v, pdns_v_l, rrname, strlen(rrname)); + + count++; + + + bucket = crc32_custom(rrname) % MOD; + sprintf(bucket_c, "%c%u", 'f', bucket); + + if (bucket < 349525) { + redisAppendCommand(c_f, "HSETNX %b %b %b", bucket_c, strlen(bucket_c), pdns_fl, pdns_fl_l, &ts, (size_t) sizeof(int)); + bucket_c[0] = 'l'; + redisAppendCommand(c_l, "HSET %b %b %b", bucket_c, strlen(bucket_c), pdns_fl, pdns_fl_l, &ts, (size_t) sizeof(int)); + count_fl++; + } else if (bucket < 349525*2) { + redisAppendCommand(c_f2, "HSETNX %b %b %b", bucket_c, strlen(bucket_c), pdns_fl, pdns_fl_l, &ts, (size_t) sizeof(int)); + bucket_c[0] = 'l'; + redisAppendCommand(c_l2, "HSET %b %b %b", bucket_c, strlen(bucket_c), pdns_fl, pdns_fl_l, &ts, (size_t) sizeof(int)); + count_fl2++; + } else { + redisAppendCommand(c_f3, "HSETNX %b %b %b", bucket_c, strlen(bucket_c), pdns_fl, pdns_fl_l, &ts, (size_t) sizeof(int)); + bucket_c[0] = 'l'; + redisAppendCommand(c_l3, "HSET %b %b %b", bucket_c, strlen(bucket_c), pdns_fl, pdns_fl_l, &ts, (size_t) sizeof(int)); + count_fl3++; + } + + if ((count % FLUSHMOD) == 0) { // {{{ + for (i=0; i> 1) & 0x55555555); + x = ((x & 0x33333333) << 2) | ((x >> 2) & 0x33333333); + x = ((x & 0x0F0F0F0F) << 4) | ((x >> 4) & 0x0F0F0F0F); + x = (x << 24) | ((x & 0xFF00) << 8) | + ((x >> 8) & 0xFF00) | (x >> 24); + return x; +} // }}} + +char *getfield(const char *s, const char del, const unsigned int n) { // {{{ + + unsigned int pos = 0; + unsigned int chars = 0; + char *start = (char *)s; + char *result; + + if (!start) + return NULL; + + while (pos != n) { + if (!*start) + return NULL; + + if (*start == del) + pos++; + + start++; + } + + while (start[chars]) { + if (start[chars] == del) + break; + chars++; + } + + if (chars) { + result = malloc(chars + 1); + memcpy(result, start, chars); + result[chars] = '\0'; + return result; + } + + return NULL; +} // }}} + +char *get2fields(const char *s, const char del1, const unsigned int n1, const char del2, const unsigned int n2) { // {{{ + if (!s) + return NULL; + + char *tmp, *tmp2; + + tmp = getfield(s, del1, n1); + + if (!tmp) { + return NULL; + } + + tmp2 = getfield(tmp, del2, n2); + free(tmp); + + if (!tmp2) + return NULL; + + return tmp2; +} // }}} + +/* + * del must not be equal to quote + */ +char *getfielddel(const char *s, const char del, const char quote, const unsigned int n) { // {{{ + + unsigned int pos = 0; + unsigned int chars = 0; + char open = 0; + char *start = (char *)s; + char *result; + + if (!start) + return NULL; + + while (pos != n) { + if (!*start) + return NULL; + + if (*start == quote) + open = !open; + + if (*start == del && !open) + pos++; + + start++; + } + + // can we do this more elegantly? + if (*start == quote) + start++; + + while (start[chars]) { + if (start[chars] == quote) //start[chars] == del) + break; + chars++; + } + + if (chars) { + result = malloc(chars + 1); + // what, if result was 0? + memcpy(result, start, chars); + result[chars] = '\0'; + return result; + } + + return NULL; + +} // }}} + diff --git a/src/redis_tools/read-parallel.sh b/src/redis_tools/read-parallel.sh new file mode 100755 index 0000000..510408a --- /dev/null +++ b/src/redis_tools/read-parallel.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +find /run/media/felix/disk/pDNS -iname 'pdns_capture.pcap-*-2017-09-0[1-7]*.csv' | sort -t- -k3 | xargs -P10 -n1 ./r-4-felix +