2014-06-18 23:05:19,250 WARN org.apache.hadoop.util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2014-06-18 23:05:19,264 WARN org.apache.hadoop.security.authentication.server.AuthenticationFilter: 'signature.secret' configuration not set, using a random value as secret
2014-06-18 23:05:20,756 WARN org.apache.solr.core.SolrResourceLoader: Can't find (or read) directory to add to classloader: ../../../contrib/extraction/lib (resolved as: /var/lib/solr/collection-log_shard2_replica1/../../../contrib/extraction/lib).
2014-06-18 23:05:20,756 WARN org.apache.solr.core.SolrResourceLoader: Can't find (or read) directory to add to classloader: ../../../dist/ (resolved as: /var/lib/solr/collection-log_shard2_replica1/../../../dist).
2014-06-18 23:05:20,756 WARN org.apache.solr.core.SolrResourceLoader: Can't find (or read) directory to add to classloader: ../../../contrib/clustering/lib/ (resolved as: /var/lib/solr/collection-log_shard2_replica1/../../../contrib/clustering/lib).
2014-06-18 23:05:20,756 WARN org.apache.solr.core.SolrResourceLoader: Can't find (or read) directory to add to classloader: ../../../dist/ (resolved as: /var/lib/solr/collection-log_shard2_replica1/../../../dist).
2014-06-18 23:05:20,756 WARN org.apache.solr.core.SolrResourceLoader: Can't find (or read) directory to add to classloader: ../../../contrib/langid/lib/ (resolved as: /var/lib/solr/collection-log_shard2_replica1/../../../contrib/langid/lib).
2014-06-18 23:05:20,757 WARN org.apache.solr.core.SolrResourceLoader: Can't find (or read) directory to add to classloader: ../../../dist/ (resolved as: /var/lib/solr/collection-log_shard2_replica1/../../../dist).
2014-06-18 23:05:20,757 WARN org.apache.solr.core.SolrResourceLoader: Can't find (or read) directory to add to classloader: ../../../contrib/velocity/lib (resolved as: /var/lib/solr/collection-log_shard2_replica1/../../../contrib/velocity/lib).
2014-06-18 23:05:20,757 WARN org.apache.solr.core.SolrResourceLoader: Can't find (or read) directory to add to classloader: ../../../dist/ (resolved as: /var/lib/solr/collection-log_shard2_replica1/../../../dist).
From To Mountain~
2014년 6월 18일 수요일
2014년 6월 4일 수요일
redis,jedis / zip stream
https://gist.github.com/allwefantasy/3615650public class RedisClient {private JedisPool pool;@Injectpublic RedisClient(Settings settings) {try {pool = new JedisPool(new JedisPoolConfig(), settings.get("redis.host"), settings.getAsInt("redis.port", 6379));} catch (SettingsException e) {// ignore}}public interface Callback {public void execute(Jedis jedis);}public void operate(Callback callback) {Jedis jedis = borrow();try {callback.execute(jedis);} finally {revert(jedis);}}public String get(String key) {Jedis jedis = borrow();try {return jedis.get(key);} finally {revert(jedis);}}public String bGet(String key) {Jedis jedis = borrow();try {byte[] value = jedis.get(key.getBytes());if (value != null) return GZip.decodeWithGZip(value);return null;} finally {revert(jedis);}}public String set(String key, String value) {Jedis jedis = borrow();try {return jedis.set(key, value);} finally {revert(jedis);}}public void del(String key) {Jedis jedis = borrow();try {jedis.del(key);} finally {revert(jedis);}}public String bSet(String key, String value) {Jedis jedis = borrow();try {return jedis.set(key.getBytes(), GZip.encodeWithGZip(value));} finally {revert(jedis);}}public byte[] bGetJ(String key) {Jedis jedis = borrow();try {byte[] value = jedis.get(key.getBytes());if (value != null) return value;return null;} finally {revert(jedis);}}public String bSetJ(String key, byte[] value) {Jedis jedis = borrow();try {return jedis.set(key.getBytes(), value);} finally {revert(jedis);}}public boolean exits(String key) {Jedis jedis = borrow();try {return jedis.exists(key);} finally {revert(jedis);}}public List<String> mget(String[] keys) {Jedis jedis = borrow();try {return jedis.mget(keys);} finally {revert(jedis);}}public String info() {Jedis jedis = borrow();try {return jedis.info();} finally {revert(jedis);}}public List<String> bMget(String[] keys) {Jedis jedis = borrow();int len = keys.length;byte[][] bkeys = new byte[len][];for (int i = 0; i < keys.length; i++) {bkeys[i] = keys[i].getBytes();}try {List<byte[]> list = jedis.mget(bkeys);List<String> temp_list = new ArrayList<String>(list.size());for (byte[] temp : list) {temp_list.add(GZip.decodeWithGZip(temp));}return temp_list;} finally {revert(jedis);}}public Set<String> sCopy(String key, String new_key) {Jedis jedis = borrow();try {Set<String> oldSets = jedis.smembers(key);for (String str : oldSets) {jedis.sadd(new_key, str);}return oldSets;} finally {revert(jedis);}}public void sClear(String key, String oldKey) {Jedis jedis = borrow();try {Set<String> oldSets = jedis.smembers(key);for (String str : oldSets) {jedis.del(oldKey + ":" + str);}jedis.del(key);} finally {revert(jedis);}}public Set<String> sMove(String key, String new_key) {Jedis jedis = borrow();try {Set<String> oldSets = jedis.smembers(key);for (String str : oldSets) {jedis.smove(key, new_key, str);}return oldSets;} finally {revert(jedis);}}public void destory() {pool.destroy();}public Jedis borrow() {return pool.getResource();}public void revert(Jedis jedis) {pool.returnResource(jedis);}}
2014년 6월 1일 일요일
hbase solr morphlines - multi type
<?xml version="1.0"?>
<indexer table="log" mapper="com.ngdata.hbaseindexer.morphline.MorphlineResultToSolrMapper">
<param name="morphlineFile" value="morphlines.conf"/>
<param name="morphlineId" value="log"/>
</indexer>
<?xml version="1.0"?>
<indexer table="dlog" mapper="com.ngdata.hbaseindexer.morphline.MorphlineResultToSolrMapper">
<param name="morphlineFile" value="morphlines.conf"/>
<param name="morphlineId" value="dlog"/>
</indexer>
------------------------------------------------------------------------------------------------------------------
---------------------------------------------------------------------------------------------------
hbase-indexer-site.xml에 대한 Lily HBase Indexer 고급 구성 스니펫(안전 밸브)
<property>
<name>hbase.zookeeper.quorum</name>
<value>host00.sylee.com,host01.sylee.com</value>
</property>
<property>
<name>hbaseindexer.zookeeper.connectstring</name>
<value>host00.sylee.com:2181</value>
</property>
---------------------------------------------------------------------------------------------------
Lily HBase Indexer 로깅 고급 구성 스니펫(안전 밸브)
log4j.logger.org.kitesdk.morphline=TRACE
log4j.logger.com.ngdata=TRACE
---------------------------------------------------------------------------------------------------
#SOLR_LOCATOR : {
# # Name of solr collection
# collection : hbase-dlog
#
# # ZooKeeper ensemble ($ZK_HOST)
# , zkHost : "host00.sylee.com,host01.sylee.com"
#}
morphlines : [
{
id : log
importCommands : ["org.kitesdk.**", "com.ngdata.**"]
commands : [
{
extractHBaseCells {
mappings : [
{
inputColumn : "cf:rowkey"
outputField : "rowkey"
type : string
source : value
}
, {
inputColumn : "cf:utc"
outputField : "utc"
type : string
source : value
}
, {
inputColumn : "cf:eqp"
outputField : "eqp"
type : string
source : value
}
]
}
}
{ logDebug { format : "output record: {}", args : ["@{}"] } }
]
}
,{
id : dlog
importCommands : ["org.kitesdk.**", "com.ngdata.**"]
commands : [
{
extractHBaseCells {
mappings : [
{
inputColumn : "cf:rowkey1"
outputField : "rowkey"
type : string
source : value
}
, {
inputColumn : "cf:eqp"
outputField : "status"
type : string
source : value
}
]
}
}
{ logDebug { format : "output record: {}", args : ["@{}"] } }
]
}
]
<indexer table="log" mapper="com.ngdata.hbaseindexer.morphline.MorphlineResultToSolrMapper">
<param name="morphlineFile" value="morphlines.conf"/>
<param name="morphlineId" value="log"/>
</indexer>
<?xml version="1.0"?>
<indexer table="dlog" mapper="com.ngdata.hbaseindexer.morphline.MorphlineResultToSolrMapper">
<param name="morphlineFile" value="morphlines.conf"/>
<param name="morphlineId" value="dlog"/>
</indexer>
------------------------------------------------------------------------------------------------------------------
---------------------------------------------------------------------------------------------------
hbase-indexer-site.xml에 대한 Lily HBase Indexer 고급 구성 스니펫(안전 밸브)
<property>
<name>hbase.zookeeper.quorum</name>
<value>host00.sylee.com,host01.sylee.com</value>
</property>
<property>
<name>hbaseindexer.zookeeper.connectstring</name>
<value>host00.sylee.com:2181</value>
</property>
---------------------------------------------------------------------------------------------------
Lily HBase Indexer 로깅 고급 구성 스니펫(안전 밸브)
log4j.logger.org.kitesdk.morphline=TRACE
log4j.logger.com.ngdata=TRACE
---------------------------------------------------------------------------------------------------
#SOLR_LOCATOR : {
# # Name of solr collection
# collection : hbase-dlog
#
# # ZooKeeper ensemble ($ZK_HOST)
# , zkHost : "host00.sylee.com,host01.sylee.com"
#}
morphlines : [
{
id : log
importCommands : ["org.kitesdk.**", "com.ngdata.**"]
commands : [
{
extractHBaseCells {
mappings : [
{
inputColumn : "cf:rowkey"
outputField : "rowkey"
type : string
source : value
}
, {
inputColumn : "cf:utc"
outputField : "utc"
type : string
source : value
}
, {
inputColumn : "cf:eqp"
outputField : "eqp"
type : string
source : value
}
]
}
}
{ logDebug { format : "output record: {}", args : ["@{}"] } }
]
}
,{
id : dlog
importCommands : ["org.kitesdk.**", "com.ngdata.**"]
commands : [
{
extractHBaseCells {
mappings : [
{
inputColumn : "cf:rowkey1"
outputField : "rowkey"
type : string
source : value
}
, {
inputColumn : "cf:eqp"
outputField : "status"
type : string
source : value
}
]
}
}
{ logDebug { format : "output record: {}", args : ["@{}"] } }
]
}
]
2014년 5월 25일 일요일
java object size & memory monitor & human display format
package everone.com.everone.fast.detector.rule;
import java.lang.instrument.Instrumentation;
public class ObjectSizeFetcher {
private static Instrumentation instrumentation;
public static void premain(String args, Instrumentation inst) {
instrumentation = inst;
}
public static long getObjectSize(Object o) {
return instrumentation.getObjectSize(o);
}
public static void main(String[] args) {
// Get current size of heap in bytes
long heapSize = Runtime.getRuntime().totalMemory();
// Get maximum size of heap in bytes. The heap cannot grow beyond this size.
// Any attempt will result in an OutOfMemoryException.
long heapMaxSize = Runtime.getRuntime().maxMemory();
// Get amount of free memory within the heap in bytes. This size will increase
// after garbage collection and decrease as new objects are created.
long heapFreeSize = Runtime.getRuntime().freeMemory();
System.out.println( "---------------" );
System.out.println("heapSize >> "+humanReadableByteCount(heapSize,true));
System.out.println("heapMaxSize >> " + humanReadableByteCount(heapMaxSize,true));
System.out.println("heapFreeSize >> " + humanReadableByteCount(heapFreeSize,true));
}
/**
* see FileUtils.byteCountToDisplaySize(long size)
* @param bytes
* @param si
* @return
*/
public static String humanReadableByteCount(long bytes, boolean si) {
int unit = si ? 1000 : 1024;
if (bytes < unit) return bytes + " B";
int exp = (int) (Math.log(bytes) / Math.log(unit));
String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp-1) + (si ? "" : "i");
return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
}
}
see
http://docs.oracle.com/javase/7/docs/api/java/lang/instrument/Instrumentation.html
import java.lang.instrument.Instrumentation;
public class ObjectSizeFetcher {
private static Instrumentation instrumentation;
public static void premain(String args, Instrumentation inst) {
instrumentation = inst;
}
public static long getObjectSize(Object o) {
return instrumentation.getObjectSize(o);
}
public static void main(String[] args) {
// Get current size of heap in bytes
long heapSize = Runtime.getRuntime().totalMemory();
// Get maximum size of heap in bytes. The heap cannot grow beyond this size.
// Any attempt will result in an OutOfMemoryException.
long heapMaxSize = Runtime.getRuntime().maxMemory();
// Get amount of free memory within the heap in bytes. This size will increase
// after garbage collection and decrease as new objects are created.
long heapFreeSize = Runtime.getRuntime().freeMemory();
System.out.println( "---------------" );
System.out.println("heapSize >> "+humanReadableByteCount(heapSize,true));
System.out.println("heapMaxSize >> " + humanReadableByteCount(heapMaxSize,true));
System.out.println("heapFreeSize >> " + humanReadableByteCount(heapFreeSize,true));
}
/**
* see FileUtils.byteCountToDisplaySize(long size)
* @param bytes
* @param si
* @return
*/
public static String humanReadableByteCount(long bytes, boolean si) {
int unit = si ? 1000 : 1024;
if (bytes < unit) return bytes + " B";
int exp = (int) (Math.log(bytes) / Math.log(unit));
String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp-1) + (si ? "" : "i");
return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
}
}
see
http://docs.oracle.com/javase/7/docs/api/java/lang/instrument/Instrumentation.html
2014년 5월 23일 금요일
linux log 설명.
* /var/log/maillog
- 메일과 관련된 로그들이 기록
* /var/log/messages
- 메일, 뉴스등을 제외한 전반적인 시스템의 모든 로그를 기록
* /var/log/secure
- 사용자들에 대한 접속을 기록
* /var/log/lastlog
- /etc/passwd 내 사용자들의 마지막으로 로그인한 정보를 기록 (lastlog 명령어로 확인 가능)
* /var/log/boot.log
- 부팅시 서비스 데몬들의 실행 상태를 기록하는 파일
* /var/log/dmesg
- 시스템이 부팅할 때 출력되는 메시지들이 기록 (dmesg 명령어로 확인 가능)
* /var/log/cron
- crontab 과 관련된 메시지들이 기록 (예약 스케쥴 작업이 정상적으로 실행되는지 확인 가능)
* /var/log/wtmp
- /etc/passwd 사용자들의 접속한 정보를 기록 (last 명령어로 확인 가능)
※ 특정 사용자 정보를 확인하려면 "last 사용자명" 을 입력하여 확인 가능(가장 최근 순으로 10개까지만 출력)
* /var/log/xferlog
- FTP 서버의 데이터 전송관련 로그를 기록 (불법파일이 전송 여부 및 전송상황의 모니터링 가능)
2014년 5월 21일 수요일
search components
Search Components
There are currently several "default" SearchComponents which now comprise the default SearchHandler. They are:
- query - QueryComponent
- facet - FacetComponent
- mlt - MoreLikeThis
- highlight - Highlighting
- debug - Debug
solr custom query parser
Custom Solr QueryParsers for fun and profit
Posted by Kelvin on 09 Sep 2013 at 02:53 pm | Tagged as: Lucene / Solr / Elastic Search / Nutch
In this post, I'll show you what you need to do to implement a custom Solr QueryParser.
Step 1
Extend QParserPlugin.
public class TestQueryParserPlugin extends QParserPlugin {
public void init(NamedList namedList) {
}
public void init(NamedList namedList) {
}
@Override public QParser createParser(String s, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
return new TestQParser(s, localParams, params, req);
}
}
return new TestQParser(s, localParams, params, req);
}
}
This is the class you'll define in solrconfig.xml, informing Solr of your queryparser. Define it like so:
<queryParser name="myfunparser"class="org.supermind.solr.queryparser.TestQParserPlugin"/>
Step 2
Extend QParser.
public class TestQParser extends QParser {
public TestQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
super(qstr, localParams, params, req);
}
public TestQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
super(qstr, localParams, params, req);
}
@Override public Query parse() throws SyntaxError {
return null;
}
}
return null;
}
}
Step 3
Actually implement the parsing in the parse() method.
Suppose we wanted to make a really simple parser for term queries, which are space-delimited. Here's how I'd do it:
@Override public Query parse() throws SyntaxError {
String defaultField = req.getSchema().getDefaultSearchFieldName();
QueryParser.Operator defaultOperator = QueryParser.Operator.valueOf(req.getSchema().getQueryParserDefaultOperator());
BooleanClause.Occur op = (defaultOperator == QueryParser.Operator.AND) ? BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD;
String[] arr = qstr.split(" ");
BooleanQuery bq = new BooleanQuery(true);
for(String s: arr) {
if(s.trim().length() == 0) continue;
bq.add(new TermQuery(new Term(defaultField, s)), op);
}
return bq;
}
String defaultField = req.getSchema().getDefaultSearchFieldName();
QueryParser.Operator defaultOperator = QueryParser.Operator.valueOf(req.getSchema().getQueryParserDefaultOperator());
BooleanClause.Occur op = (defaultOperator == QueryParser.Operator.AND) ? BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD;
String[] arr = qstr.split(" ");
BooleanQuery bq = new BooleanQuery(true);
for(String s: arr) {
if(s.trim().length() == 0) continue;
bq.add(new TermQuery(new Term(defaultField, s)), op);
}
return bq;
}
Step 4
In your query, use the nested query syntax to call your queryparser, e.g.
http://localhost:8983/solr/collection1/select?q={!myfunparser}foo+bar+car
Maybe in a follow-up post, I'll post the full code with jars and all.
피드 구독하기:
글 (Atom)