Search Results

Search found 896 results on 36 pages for 'analyze'.

Page 1/36 | 1 2 3 4 5 6 7 8 9 10 11 12  | Next Page >

  • Using R to Analyze G1GC Log Files

    - by user12620111
    Using R to Analyze G1GC Log Files body, td { font-family: sans-serif; background-color: white; font-size: 12px; margin: 8px; } tt, code, pre { font-family: 'DejaVu Sans Mono', 'Droid Sans Mono', 'Lucida Console', Consolas, Monaco, monospace; } h1 { font-size:2.2em; } h2 { font-size:1.8em; } h3 { font-size:1.4em; } h4 { font-size:1.0em; } h5 { font-size:0.9em; } h6 { font-size:0.8em; } a:visited { color: rgb(50%, 0%, 50%); } pre { margin-top: 0; max-width: 95%; border: 1px solid #ccc; white-space: pre-wrap; } pre code { display: block; padding: 0.5em; } code.r, code.cpp { background-color: #F8F8F8; } table, td, th { border: none; } blockquote { color:#666666; margin:0; padding-left: 1em; border-left: 0.5em #EEE solid; } hr { height: 0px; border-bottom: none; border-top-width: thin; border-top-style: dotted; border-top-color: #999999; } @media print { * { background: transparent !important; color: black !important; filter:none !important; -ms-filter: none !important; } body { font-size:12pt; max-width:100%; } a, a:visited { text-decoration: underline; } hr { visibility: hidden; page-break-before: always; } pre, blockquote { padding-right: 1em; page-break-inside: avoid; } tr, img { page-break-inside: avoid; } img { max-width: 100% !important; } @page :left { margin: 15mm 20mm 15mm 10mm; } @page :right { margin: 15mm 10mm 15mm 20mm; } p, h2, h3 { orphans: 3; widows: 3; } h2, h3 { page-break-after: avoid; } } pre .operator, pre .paren { color: rgb(104, 118, 135) } pre .literal { color: rgb(88, 72, 246) } pre .number { color: rgb(0, 0, 205); } pre .comment { color: rgb(76, 136, 107); } pre .keyword { color: rgb(0, 0, 255); } pre .identifier { color: rgb(0, 0, 0); } pre .string { color: rgb(3, 106, 7); } var hljs=new function(){function m(p){return p.replace(/&/gm,"&").replace(/"}while(y.length||w.length){var v=u().splice(0,1)[0];z+=m(x.substr(q,v.offset-q));q=v.offset;if(v.event=="start"){z+=t(v.node);s.push(v.node)}else{if(v.event=="stop"){var p,r=s.length;do{r--;p=s[r];z+=("")}while(p!=v.node);s.splice(r,1);while(r'+M[0]+""}else{r+=M[0]}O=P.lR.lastIndex;M=P.lR.exec(L)}return r+L.substr(O,L.length-O)}function J(L,M){if(M.sL&&e[M.sL]){var r=d(M.sL,L);x+=r.keyword_count;return r.value}else{return F(L,M)}}function I(M,r){var L=M.cN?'':"";if(M.rB){y+=L;M.buffer=""}else{if(M.eB){y+=m(r)+L;M.buffer=""}else{y+=L;M.buffer=r}}D.push(M);A+=M.r}function G(N,M,Q){var R=D[D.length-1];if(Q){y+=J(R.buffer+N,R);return false}var P=q(M,R);if(P){y+=J(R.buffer+N,R);I(P,M);return P.rB}var L=v(D.length-1,M);if(L){var O=R.cN?"":"";if(R.rE){y+=J(R.buffer+N,R)+O}else{if(R.eE){y+=J(R.buffer+N,R)+O+m(M)}else{y+=J(R.buffer+N+M,R)+O}}while(L1){O=D[D.length-2].cN?"":"";y+=O;L--;D.length--}var r=D[D.length-1];D.length--;D[D.length-1].buffer="";if(r.starts){I(r.starts,"")}return R.rE}if(w(M,R)){throw"Illegal"}}var E=e[B];var D=[E.dM];var A=0;var x=0;var y="";try{var s,u=0;E.dM.buffer="";do{s=p(C,u);var t=G(s[0],s[1],s[2]);u+=s[0].length;if(!t){u+=s[1].length}}while(!s[2]);if(D.length1){throw"Illegal"}return{r:A,keyword_count:x,value:y}}catch(H){if(H=="Illegal"){return{r:0,keyword_count:0,value:m(C)}}else{throw H}}}function g(t){var p={keyword_count:0,r:0,value:m(t)};var r=p;for(var q in e){if(!e.hasOwnProperty(q)){continue}var s=d(q,t);s.language=q;if(s.keyword_count+s.rr.keyword_count+r.r){r=s}if(s.keyword_count+s.rp.keyword_count+p.r){r=p;p=s}}if(r.language){p.second_best=r}return p}function i(r,q,p){if(q){r=r.replace(/^((]+|\t)+)/gm,function(t,w,v,u){return w.replace(/\t/g,q)})}if(p){r=r.replace(/\n/g,"")}return r}function n(t,w,r){var x=h(t,r);var v=a(t);var y,s;if(v){y=d(v,x)}else{return}var q=c(t);if(q.length){s=document.createElement("pre");s.innerHTML=y.value;y.value=k(q,c(s),x)}y.value=i(y.value,w,r);var u=t.className;if(!u.match("(\\s|^)(language-)?"+v+"(\\s|$)")){u=u?(u+" "+v):v}if(/MSIE [678]/.test(navigator.userAgent)&&t.tagName=="CODE"&&t.parentNode.tagName=="PRE"){s=t.parentNode;var p=document.createElement("div");p.innerHTML=""+y.value+"";t=p.firstChild.firstChild;p.firstChild.cN=s.cN;s.parentNode.replaceChild(p.firstChild,s)}else{t.innerHTML=y.value}t.className=u;t.result={language:v,kw:y.keyword_count,re:y.r};if(y.second_best){t.second_best={language:y.second_best.language,kw:y.second_best.keyword_count,re:y.second_best.r}}}function o(){if(o.called){return}o.called=true;var r=document.getElementsByTagName("pre");for(var p=0;p|=||=||=|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~";this.ER="(?![\\s\\S])";this.BE={b:"\\\\.",r:0};this.ASM={cN:"string",b:"'",e:"'",i:"\\n",c:[this.BE],r:0};this.QSM={cN:"string",b:'"',e:'"',i:"\\n",c:[this.BE],r:0};this.CLCM={cN:"comment",b:"//",e:"$"};this.CBLCLM={cN:"comment",b:"/\\*",e:"\\*/"};this.HCM={cN:"comment",b:"#",e:"$"};this.NM={cN:"number",b:this.NR,r:0};this.CNM={cN:"number",b:this.CNR,r:0};this.BNM={cN:"number",b:this.BNR,r:0};this.inherit=function(r,s){var p={};for(var q in r){p[q]=r[q]}if(s){for(var q in s){p[q]=s[q]}}return p}}();hljs.LANGUAGES.cpp=function(){var a={keyword:{"false":1,"int":1,"float":1,"while":1,"private":1,"char":1,"catch":1,"export":1,virtual:1,operator:2,sizeof:2,dynamic_cast:2,typedef:2,const_cast:2,"const":1,struct:1,"for":1,static_cast:2,union:1,namespace:1,unsigned:1,"long":1,"throw":1,"volatile":2,"static":1,"protected":1,bool:1,template:1,mutable:1,"if":1,"public":1,friend:2,"do":1,"return":1,"goto":1,auto:1,"void":2,"enum":1,"else":1,"break":1,"new":1,extern:1,using:1,"true":1,"class":1,asm:1,"case":1,typeid:1,"short":1,reinterpret_cast:2,"default":1,"double":1,register:1,explicit:1,signed:1,typename:1,"try":1,"this":1,"switch":1,"continue":1,wchar_t:1,inline:1,"delete":1,alignof:1,char16_t:1,char32_t:1,constexpr:1,decltype:1,noexcept:1,nullptr:1,static_assert:1,thread_local:1,restrict:1,_Bool:1,complex:1},built_in:{std:1,string:1,cin:1,cout:1,cerr:1,clog:1,stringstream:1,istringstream:1,ostringstream:1,auto_ptr:1,deque:1,list:1,queue:1,stack:1,vector:1,map:1,set:1,bitset:1,multiset:1,multimap:1,unordered_set:1,unordered_map:1,unordered_multiset:1,unordered_multimap:1,array:1,shared_ptr:1}};return{dM:{k:a,i:"",k:a,r:10,c:["self"]}]}}}();hljs.LANGUAGES.r={dM:{c:[hljs.HCM,{cN:"number",b:"\\b0[xX][0-9a-fA-F]+[Li]?\\b",e:hljs.IMMEDIATE_RE,r:0},{cN:"number",b:"\\b\\d+(?:[eE][+\\-]?\\d*)?L\\b",e:hljs.IMMEDIATE_RE,r:0},{cN:"number",b:"\\b\\d+\\.(?!\\d)(?:i\\b)?",e:hljs.IMMEDIATE_RE,r:1},{cN:"number",b:"\\b\\d+(?:\\.\\d*)?(?:[eE][+\\-]?\\d*)?i?\\b",e:hljs.IMMEDIATE_RE,r:0},{cN:"number",b:"\\.\\d+(?:[eE][+\\-]?\\d*)?i?\\b",e:hljs.IMMEDIATE_RE,r:1},{cN:"keyword",b:"(?:tryCatch|library|setGeneric|setGroupGeneric)\\b",e:hljs.IMMEDIATE_RE,r:10},{cN:"keyword",b:"\\.\\.\\.",e:hljs.IMMEDIATE_RE,r:10},{cN:"keyword",b:"\\.\\.\\d+(?![\\w.])",e:hljs.IMMEDIATE_RE,r:10},{cN:"keyword",b:"\\b(?:function)",e:hljs.IMMEDIATE_RE,r:2},{cN:"keyword",b:"(?:if|in|break|next|repeat|else|for|return|switch|while|try|stop|warning|require|attach|detach|source|setMethod|setClass)\\b",e:hljs.IMMEDIATE_RE,r:1},{cN:"literal",b:"(?:NA|NA_integer_|NA_real_|NA_character_|NA_complex_)\\b",e:hljs.IMMEDIATE_RE,r:10},{cN:"literal",b:"(?:NULL|TRUE|FALSE|T|F|Inf|NaN)\\b",e:hljs.IMMEDIATE_RE,r:1},{cN:"identifier",b:"[a-zA-Z.][a-zA-Z0-9._]*\\b",e:hljs.IMMEDIATE_RE,r:0},{cN:"operator",b:"|=||   Using R to Analyze G1GC Log Files   Using R to Analyze G1GC Log Files Introduction Working in Oracle Platform Integration gives an engineer opportunities to work on a wide array of technologies. My team’s goal is to make Oracle applications run best on the Solaris/SPARC platform. When looking for bottlenecks in a modern applications, one needs to be aware of not only how the CPUs and operating system are executing, but also network, storage, and in some cases, the Java Virtual Machine. I was recently presented with about 1.5 GB of Java Garbage First Garbage Collector log file data. If you’re not familiar with the subject, you might want to review Garbage First Garbage Collector Tuning by Monica Beckwith. The customer had been running Java HotSpot 1.6.0_31 to host a web application server. I was told that the Solaris/SPARC server was running a Java process launched using a commmand line that included the following flags: -d64 -Xms9g -Xmx9g -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:InitiatingHeapOccupancyPercent=80 -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+PrintGC -XX:+PrintGCTimeStamps -XX:+PrintHeapAtGC -XX:+PrintGCDateStamps -XX:+PrintFlagsFinal -XX:+DisableExplicitGC -XX:+UnlockExperimentalVMOptions -XX:ParallelGCThreads=8 Several sources on the internet indicate that if I were to print out the 1.5 GB of log files, it would require enough paper to fill the bed of a pick up truck. Of course, it would be fruitless to try to scan the log files by hand. Tools will be required to summarize the contents of the log files. Others have encountered large Java garbage collection log files. There are existing tools to analyze the log files: IBM’s GC toolkit The chewiebug GCViewer gchisto HPjmeter Instead of using one of the other tools listed, I decide to parse the log files with standard Unix tools, and analyze the data with R. Data Cleansing The log files arrived in two different formats. I guess that the difference is that one set of log files was generated using a more verbose option, maybe -XX:+PrintHeapAtGC, and the other set of log files was generated without that option. Format 1 In some of the log files, the log files with the less verbose format, a single trace, i.e. the report of a singe garbage collection event, looks like this: {Heap before GC invocations=12280 (full 61): garbage-first heap total 9437184K, used 7499918K [0xfffffffd00000000, 0xffffffff40000000, 0xffffffff40000000) region size 4096K, 1 young (4096K), 0 survivors (0K) compacting perm gen total 262144K, used 144077K [0xffffffff40000000, 0xffffffff50000000, 0xffffffff50000000) the space 262144K, 54% used [0xffffffff40000000, 0xffffffff48cb3758, 0xffffffff48cb3800, 0xffffffff50000000) No shared spaces configured. 2014-05-14T07:24:00.988-0700: 60586.353: [GC pause (young) 7324M->7320M(9216M), 0.1567265 secs] Heap after GC invocations=12281 (full 61): garbage-first heap total 9437184K, used 7496533K [0xfffffffd00000000, 0xffffffff40000000, 0xffffffff40000000) region size 4096K, 0 young (0K), 0 survivors (0K) compacting perm gen total 262144K, used 144077K [0xffffffff40000000, 0xffffffff50000000, 0xffffffff50000000) the space 262144K, 54% used [0xffffffff40000000, 0xffffffff48cb3758, 0xffffffff48cb3800, 0xffffffff50000000) No shared spaces configured. } A simple grep can be used to extract a summary: $ grep "\[ GC pause (young" g1gc.log 2014-05-13T13:24:35.091-0700: 3.109: [GC pause (young) 20M->5029K(9216M), 0.0146328 secs] 2014-05-13T13:24:35.440-0700: 3.459: [GC pause (young) 9125K->6077K(9216M), 0.0086723 secs] 2014-05-13T13:24:37.581-0700: 5.599: [GC pause (young) 25M->8470K(9216M), 0.0203820 secs] 2014-05-13T13:24:42.686-0700: 10.704: [GC pause (young) 44M->15M(9216M), 0.0288848 secs] 2014-05-13T13:24:48.941-0700: 16.958: [GC pause (young) 51M->20M(9216M), 0.0491244 secs] 2014-05-13T13:24:56.049-0700: 24.066: [GC pause (young) 92M->26M(9216M), 0.0525368 secs] 2014-05-13T13:25:34.368-0700: 62.383: [GC pause (young) 602M->68M(9216M), 0.1721173 secs] But that format wasn't easily read into R, so I needed to be a bit more tricky. I used the following Unix command to create a summary file that was easy for R to read. $ echo "SecondsSinceLaunch BeforeSize AfterSize TotalSize RealTime" $ grep "\[GC pause (young" g1gc.log | grep -v mark | sed -e 's/[A-SU-z\(\),]/ /g' -e 's/->/ /' -e 's/: / /g' | more SecondsSinceLaunch BeforeSize AfterSize TotalSize RealTime 2014-05-13T13:24:35.091-0700 3.109 20 5029 9216 0.0146328 2014-05-13T13:24:35.440-0700 3.459 9125 6077 9216 0.0086723 2014-05-13T13:24:37.581-0700 5.599 25 8470 9216 0.0203820 2014-05-13T13:24:42.686-0700 10.704 44 15 9216 0.0288848 2014-05-13T13:24:48.941-0700 16.958 51 20 9216 0.0491244 2014-05-13T13:24:56.049-0700 24.066 92 26 9216 0.0525368 2014-05-13T13:25:34.368-0700 62.383 602 68 9216 0.1721173 Format 2 In some of the log files, the log files with the more verbose format, a single trace, i.e. the report of a singe garbage collection event, was more complicated than Format 1. Here is a text file with an example of a single G1GC trace in the second format. As you can see, it is quite complicated. It is nice that there is so much information available, but the level of detail can be overwhelming. I wrote this awk script (download) to summarize each trace on a single line. #!/usr/bin/env awk -f BEGIN { printf("SecondsSinceLaunch IncrementalCount FullCount UserTime SysTime RealTime BeforeSize AfterSize TotalSize\n") } ###################### # Save count data from lines that are at the start of each G1GC trace. # Each trace starts out like this: # {Heap before GC invocations=14 (full 0): # garbage-first heap total 9437184K, used 325496K [0xfffffffd00000000, 0xffffffff40000000, 0xffffffff40000000) ###################### /{Heap.*full/{ gsub ( "\\)" , "" ); nf=split($0,a,"="); split(a[2],b," "); getline; if ( match($0, "first") ) { G1GC=1; IncrementalCount=b[1]; FullCount=substr( b[3], 1, length(b[3])-1 ); } else { G1GC=0; } } ###################### # Pull out time stamps that are in lines with this format: # 2014-05-12T14:02:06.025-0700: 94.312: [GC pause (young), 0.08870154 secs] ###################### /GC pause/ { DateTime=$1; SecondsSinceLaunch=substr($2, 1, length($2)-1); } ###################### # Heap sizes are in lines that look like this: # [ 4842M->4838M(9216M)] ###################### /\[ .*]$/ { gsub ( "\\[" , "" ); gsub ( "\ \]" , "" ); gsub ( "->" , " " ); gsub ( "\\( " , " " ); gsub ( "\ \)" , " " ); split($0,a," "); if ( split(a[1],b,"M") > 1 ) {BeforeSize=b[1]*1024;} if ( split(a[1],b,"K") > 1 ) {BeforeSize=b[1];} if ( split(a[2],b,"M") > 1 ) {AfterSize=b[1]*1024;} if ( split(a[2],b,"K") > 1 ) {AfterSize=b[1];} if ( split(a[3],b,"M") > 1 ) {TotalSize=b[1]*1024;} if ( split(a[3],b,"K") > 1 ) {TotalSize=b[1];} } ###################### # Emit an output line when you find input that looks like this: # [Times: user=1.41 sys=0.08, real=0.24 secs] ###################### /\[Times/ { if (G1GC==1) { gsub ( "," , "" ); split($2,a,"="); UserTime=a[2]; split($3,a,"="); SysTime=a[2]; split($4,a,"="); RealTime=a[2]; print DateTime,SecondsSinceLaunch,IncrementalCount,FullCount,UserTime,SysTime,RealTime,BeforeSize,AfterSize,TotalSize; G1GC=0; } } The resulting summary is about 25X smaller that the original file, but still difficult for a human to digest. SecondsSinceLaunch IncrementalCount FullCount UserTime SysTime RealTime BeforeSize AfterSize TotalSize ... 2014-05-12T18:36:34.669-0700: 3985.744 561 0 0.57 0.06 0.16 1724416 1720320 9437184 2014-05-12T18:36:34.839-0700: 3985.914 562 0 0.51 0.06 0.19 1724416 1720320 9437184 2014-05-12T18:36:35.069-0700: 3986.144 563 0 0.60 0.04 0.27 1724416 1721344 9437184 2014-05-12T18:36:35.354-0700: 3986.429 564 0 0.33 0.04 0.09 1725440 1722368 9437184 2014-05-12T18:36:35.545-0700: 3986.620 565 0 0.58 0.04 0.17 1726464 1722368 9437184 2014-05-12T18:36:35.726-0700: 3986.801 566 0 0.43 0.05 0.12 1726464 1722368 9437184 2014-05-12T18:36:35.856-0700: 3986.930 567 0 0.30 0.04 0.07 1726464 1723392 9437184 2014-05-12T18:36:35.947-0700: 3987.023 568 0 0.61 0.04 0.26 1727488 1723392 9437184 2014-05-12T18:36:36.228-0700: 3987.302 569 0 0.46 0.04 0.16 1731584 1724416 9437184 Reading the Data into R Once the GC log data had been cleansed, either by processing the first format with the shell script, or by processing the second format with the awk script, it was easy to read the data into R. g1gc.df = read.csv("summary.txt", row.names = NULL, stringsAsFactors=FALSE,sep="") str(g1gc.df) ## 'data.frame': 8307 obs. of 10 variables: ## $ row.names : chr "2014-05-12T14:00:32.868-0700:" "2014-05-12T14:00:33.179-0700:" "2014-05-12T14:00:33.677-0700:" "2014-05-12T14:00:35.538-0700:" ... ## $ SecondsSinceLaunch: num 1.16 1.47 1.97 3.83 6.1 ... ## $ IncrementalCount : int 0 1 2 3 4 5 6 7 8 9 ... ## $ FullCount : int 0 0 0 0 0 0 0 0 0 0 ... ## $ UserTime : num 0.11 0.05 0.04 0.21 0.08 0.26 0.31 0.33 0.34 0.56 ... ## $ SysTime : num 0.04 0.01 0.01 0.05 0.01 0.06 0.07 0.06 0.07 0.09 ... ## $ RealTime : num 0.02 0.02 0.01 0.04 0.02 0.04 0.05 0.04 0.04 0.06 ... ## $ BeforeSize : int 8192 5496 5768 22528 24576 43008 34816 53248 55296 93184 ... ## $ AfterSize : int 1400 1672 2557 4907 7072 14336 16384 18432 19456 21504 ... ## $ TotalSize : int 9437184 9437184 9437184 9437184 9437184 9437184 9437184 9437184 9437184 9437184 ... head(g1gc.df) ## row.names SecondsSinceLaunch IncrementalCount ## 1 2014-05-12T14:00:32.868-0700: 1.161 0 ## 2 2014-05-12T14:00:33.179-0700: 1.472 1 ## 3 2014-05-12T14:00:33.677-0700: 1.969 2 ## 4 2014-05-12T14:00:35.538-0700: 3.830 3 ## 5 2014-05-12T14:00:37.811-0700: 6.103 4 ## 6 2014-05-12T14:00:41.428-0700: 9.720 5 ## FullCount UserTime SysTime RealTime BeforeSize AfterSize TotalSize ## 1 0 0.11 0.04 0.02 8192 1400 9437184 ## 2 0 0.05 0.01 0.02 5496 1672 9437184 ## 3 0 0.04 0.01 0.01 5768 2557 9437184 ## 4 0 0.21 0.05 0.04 22528 4907 9437184 ## 5 0 0.08 0.01 0.02 24576 7072 9437184 ## 6 0 0.26 0.06 0.04 43008 14336 9437184 Basic Statistics Once the data has been read into R, simple statistics are very easy to generate. All of the numbers from high school statistics are available via simple commands. For example, generate a summary of every column: summary(g1gc.df) ## row.names SecondsSinceLaunch IncrementalCount FullCount ## Length:8307 Min. : 1 Min. : 0 Min. : 0.0 ## Class :character 1st Qu.: 9977 1st Qu.:2048 1st Qu.: 0.0 ## Mode :character Median :12855 Median :4136 Median : 12.0 ## Mean :12527 Mean :4156 Mean : 31.6 ## 3rd Qu.:15758 3rd Qu.:6262 3rd Qu.: 61.0 ## Max. :55484 Max. :8391 Max. :113.0 ## UserTime SysTime RealTime BeforeSize ## Min. :0.040 Min. :0.0000 Min. : 0.0 Min. : 5476 ## 1st Qu.:0.470 1st Qu.:0.0300 1st Qu.: 0.1 1st Qu.:5137920 ## Median :0.620 Median :0.0300 Median : 0.1 Median :6574080 ## Mean :0.751 Mean :0.0355 Mean : 0.3 Mean :5841855 ## 3rd Qu.:0.920 3rd Qu.:0.0400 3rd Qu.: 0.2 3rd Qu.:7084032 ## Max. :3.370 Max. :1.5600 Max. :488.1 Max. :8696832 ## AfterSize TotalSize ## Min. : 1380 Min. :9437184 ## 1st Qu.:5002752 1st Qu.:9437184 ## Median :6559744 Median :9437184 ## Mean :5785454 Mean :9437184 ## 3rd Qu.:7054336 3rd Qu.:9437184 ## Max. :8482816 Max. :9437184 Q: What is the total amount of User CPU time spent in garbage collection? sum(g1gc.df$UserTime) ## [1] 6236 As you can see, less than two hours of CPU time was spent in garbage collection. Is that too much? To find the percentage of time spent in garbage collection, divide the number above by total_elapsed_time*CPU_count. In this case, there are a lot of CPU’s and it turns out the the overall amount of CPU time spent in garbage collection isn’t a problem when viewed in isolation. When calculating rates, i.e. events per unit time, you need to ask yourself if the rate is homogenous across the time period in the log file. Does the log file include spikes of high activity that should be separately analyzed? Averaging in data from nights and weekends with data from business hours may alias problems. If you have a reason to suspect that the garbage collection rates include peaks and valleys that need independent analysis, see the “Time Series” section, below. Q: How much garbage is collected on each pass? The amount of heap space that is recovered per GC pass is surprisingly low: At least one collection didn’t recover any data. (“Min.=0”) 25% of the passes recovered 3MB or less. (“1st Qu.=3072”) Half of the GC passes recovered 4MB or less. (“Median=4096”) The average amount recovered was 56MB. (“Mean=56390”) 75% of the passes recovered 36MB or less. (“3rd Qu.=36860”) At least one pass recovered 2GB. (“Max.=2121000”) g1gc.df$Delta = g1gc.df$BeforeSize - g1gc.df$AfterSize summary(g1gc.df$Delta) ## Min. 1st Qu. Median Mean 3rd Qu. Max. ## 0 3070 4100 56400 36900 2120000 Q: What is the maximum User CPU time for a single collection? The worst garbage collection (“Max.”) is many standard deviations away from the mean. The data appears to be right skewed. summary(g1gc.df$UserTime) ## Min. 1st Qu. Median Mean 3rd Qu. Max. ## 0.040 0.470 0.620 0.751 0.920 3.370 sd(g1gc.df$UserTime) ## [1] 0.3966 Basic Graphics Once the data is in R, it is trivial to plot the data with formats including dot plots, line charts, bar charts (simple, stacked, grouped), pie charts, boxplots, scatter plots histograms, and kernel density plots. Histogram of User CPU Time per Collection I don't think that this graph requires any explanation. hist(g1gc.df$UserTime, main="User CPU Time per Collection", xlab="Seconds", ylab="Frequency") Box plot to identify outliers When the initial data is viewed with a box plot, you can see the one crazy outlier in the real time per GC. Save this data point for future analysis and drop the outlier so that it’s not throwing off our statistics. Now the box plot shows many outliers, which will be examined later, using times series analysis. Notice that the scale of the x-axis changes drastically once the crazy outlier is removed. par(mfrow=c(2,1)) boxplot(g1gc.df$UserTime,g1gc.df$SysTime,g1gc.df$RealTime, main="Box Plot of Time per GC\n(dominated by a crazy outlier)", names=c("usr","sys","elapsed"), xlab="Seconds per GC", ylab="Time (Seconds)", horizontal = TRUE, outcol="red") crazy.outlier.df=g1gc.df[g1gc.df$RealTime > 400,] g1gc.df=g1gc.df[g1gc.df$RealTime < 400,] boxplot(g1gc.df$UserTime,g1gc.df$SysTime,g1gc.df$RealTime, main="Box Plot of Time per GC\n(crazy outlier excluded)", names=c("usr","sys","elapsed"), xlab="Seconds per GC", ylab="Time (Seconds)", horizontal = TRUE, outcol="red") box(which = "outer", lty = "solid") Here is the crazy outlier for future analysis: crazy.outlier.df ## row.names SecondsSinceLaunch IncrementalCount ## 8233 2014-05-12T23:15:43.903-0700: 20741 8316 ## FullCount UserTime SysTime RealTime BeforeSize AfterSize TotalSize ## 8233 112 0.55 0.42 488.1 8381440 8235008 9437184 ## Delta ## 8233 146432 R Time Series Data To analyze the garbage collection as a time series, I’ll use Z’s Ordered Observations (zoo). “zoo is the creator for an S3 class of indexed totally ordered observations which includes irregular time series.” require(zoo) ## Loading required package: zoo ## ## Attaching package: 'zoo' ## ## The following objects are masked from 'package:base': ## ## as.Date, as.Date.numeric head(g1gc.df[,1]) ## [1] "2014-05-12T14:00:32.868-0700:" "2014-05-12T14:00:33.179-0700:" ## [3] "2014-05-12T14:00:33.677-0700:" "2014-05-12T14:00:35.538-0700:" ## [5] "2014-05-12T14:00:37.811-0700:" "2014-05-12T14:00:41.428-0700:" options("digits.secs"=3) times=as.POSIXct( g1gc.df[,1], format="%Y-%m-%dT%H:%M:%OS%z:") g1gc.z = zoo(g1gc.df[,-c(1)], order.by=times) head(g1gc.z) ## SecondsSinceLaunch IncrementalCount FullCount ## 2014-05-12 17:00:32.868 1.161 0 0 ## 2014-05-12 17:00:33.178 1.472 1 0 ## 2014-05-12 17:00:33.677 1.969 2 0 ## 2014-05-12 17:00:35.538 3.830 3 0 ## 2014-05-12 17:00:37.811 6.103 4 0 ## 2014-05-12 17:00:41.427 9.720 5 0 ## UserTime SysTime RealTime BeforeSize AfterSize ## 2014-05-12 17:00:32.868 0.11 0.04 0.02 8192 1400 ## 2014-05-12 17:00:33.178 0.05 0.01 0.02 5496 1672 ## 2014-05-12 17:00:33.677 0.04 0.01 0.01 5768 2557 ## 2014-05-12 17:00:35.538 0.21 0.05 0.04 22528 4907 ## 2014-05-12 17:00:37.811 0.08 0.01 0.02 24576 7072 ## 2014-05-12 17:00:41.427 0.26 0.06 0.04 43008 14336 ## TotalSize Delta ## 2014-05-12 17:00:32.868 9437184 6792 ## 2014-05-12 17:00:33.178 9437184 3824 ## 2014-05-12 17:00:33.677 9437184 3211 ## 2014-05-12 17:00:35.538 9437184 17621 ## 2014-05-12 17:00:37.811 9437184 17504 ## 2014-05-12 17:00:41.427 9437184 28672 Example of Two Benchmark Runs in One Log File The data in the following graph is from a different log file, not the one of primary interest to this article. I’m including this image because it is an example of idle periods followed by busy periods. It would be uninteresting to average the rate of garbage collection over the entire log file period. More interesting would be the rate of garbage collect in the two busy periods. Are they the same or different? Your production data may be similar, for example, bursts when employees return from lunch and idle times on weekend evenings, etc. Once the data is in an R Time Series, you can analyze isolated time windows. Clipping the Time Series data Flashing back to our test case… Viewing the data as a time series is interesting. You can see that the work intensive time period is between 9:00 PM and 3:00 AM. Lets clip the data to the interesting period:     par(mfrow=c(2,1)) plot(g1gc.z$UserTime, type="h", main="User Time per GC\nTime: Complete Log File", xlab="Time of Day", ylab="CPU Seconds per GC", col="#1b9e77") clipped.g1gc.z=window(g1gc.z, start=as.POSIXct("2014-05-12 21:00:00"), end=as.POSIXct("2014-05-13 03:00:00")) plot(clipped.g1gc.z$UserTime, type="h", main="User Time per GC\nTime: Limited to Benchmark Execution", xlab="Time of Day", ylab="CPU Seconds per GC", col="#1b9e77") box(which = "outer", lty = "solid") Cumulative Incremental and Full GC count Here is the cumulative incremental and full GC count. When the line is very steep, it indicates that the GCs are repeating very quickly. Notice that the scale on the Y axis is different for full vs. incremental. plot(clipped.g1gc.z[,c(2:3)], main="Cumulative Incremental and Full GC count", xlab="Time of Day", col="#1b9e77") GC Analysis of Benchmark Execution using Time Series data In the following series of 3 graphs: The “After Size” show the amount of heap space in use after each garbage collection. Many Java objects are still referenced, i.e. alive, during each garbage collection. This may indicate that the application has a memory leak, or may indicate that the application has a very large memory footprint. Typically, an application's memory footprint plateau's in the early stage of execution. One would expect this graph to have a flat top. The steep decline in the heap space may indicate that the application crashed after 2:00. The second graph shows that the outliers in real execution time, discussed above, occur near 2:00. when the Java heap seems to be quite full. The third graph shows that Full GCs are infrequent during the first few hours of execution. The rate of Full GC's, (the slope of the cummulative Full GC line), changes near midnight.   plot(clipped.g1gc.z[,c("AfterSize","RealTime","FullCount")], xlab="Time of Day", col=c("#1b9e77","red","#1b9e77")) GC Analysis of heap recovered Each GC trace includes the amount of heap space in use before and after the individual GC event. During garbage coolection, unreferenced objects are identified, the space holding the unreferenced objects is freed, and thus, the difference in before and after usage indicates how much space has been freed. The following box plot and bar chart both demonstrate the same point - the amount of heap space freed per garbage colloection is surprisingly low. par(mfrow=c(2,1)) boxplot(as.vector(clipped.g1gc.z$Delta), main="Amount of Heap Recovered per GC Pass", xlab="Size in KB", horizontal = TRUE, col="red") hist(as.vector(clipped.g1gc.z$Delta), main="Amount of Heap Recovered per GC Pass", xlab="Size in KB", breaks=100, col="red") box(which = "outer", lty = "solid") This graph is the most interesting. The dark blue area shows how much heap is occupied by referenced Java objects. This represents memory that holds live data. The red fringe at the top shows how much data was recovered after each garbage collection. barplot(clipped.g1gc.z[,c("AfterSize","Delta")], col=c("#7570b3","#e7298a"), xlab="Time of Day", border=NA) legend("topleft", c("Live Objects","Heap Recovered on GC"), fill=c("#7570b3","#e7298a")) box(which = "outer", lty = "solid") When I discuss the data in the log files with the customer, I will ask for an explaination for the large amount of referenced data resident in the Java heap. There are two are posibilities: There is a memory leak and the amount of space required to hold referenced objects will continue to grow, limited only by the maximum heap size. After the maximum heap size is reached, the JVM will throw an “Out of Memory” exception every time that the application tries to allocate a new object. If this is the case, the aplication needs to be debugged to identify why old objects are referenced when they are no longer needed. The application has a legitimate requirement to keep a large amount of data in memory. The customer may want to further increase the maximum heap size. Another possible solution would be to partition the application across multiple cluster nodes, where each node has responsibility for managing a unique subset of the data. Conclusion In conclusion, R is a very powerful tool for the analysis of Java garbage collection log files. The primary difficulty is data cleansing so that information can be read into an R data frame. Once the data has been read into R, a rich set of tools may be used for thorough evaluation.

    Read the article

  • Automatically analyze excel files

    - by dole doug
    I have to replicate a manual generation of a large number of excel files. I started to manually track the relations between cells ( files, formulas, etc). I also had a talk with the person which generates those files. For now I have a general understanding about how the excel files are generated, but "devil is in the details". I assume that I can write a script which will generate the hierarchy between cells and files, but this might require the same effort as manually noticing the relations. Also, I'm afraid that I'm not too experienced and my app is more prone to error approach than a manual analyze. How to handle this problem? Do you know about an open source project which analyze the excel files in a recursive mode following the formulas ?

    Read the article

  • Globacom and mCentric Deploy BDA and NoSQL Database to analyze network traffic 40x faster

    - by Jean-Pierre Dijcks
    In a fast evolving market, speed is of the essence. mCentric and Globacom leveraged Big Data Appliance, Oracle NoSQL Database to save over 35,000 Call-Processing minutes daily and analyze network traffic 40x faster.  Here are some highlights from the profile: Why Oracle “Oracle Big Data Appliance works well for very large amounts of structured and unstructured data. It is the most agile events-storage system for our collect-it-now and analyze-it-later set of business requirements. Moreover, choosing a prebuilt solution drastically reduced implementation time. We got the big data benefits without needing to assemble and tune a custom-built system, and without the hidden costs required to maintain a large number of servers in our data center. A single support license covers both the hardware and the integrated software, and we have one central point of contact for support,” said Sanjib Roy, CTO, Globacom. Implementation Process It took only five days for Oracle partner mCentric to deploy Oracle Big Data Appliance, perform the software install and configuration, certification, and resiliency testing. The entire process—from site planning to phase-I, go-live—was executed in just over ten weeks, well ahead of the four months allocated to complete the project. Oracle partner mCentric leveraged Oracle Advanced Customer Support Services’ implementation methodology to ensure configurations are tailored for peak performance, all patches are applied, and software and communications are consistently tested using proven methodologies and best practices. Read the entire profile here.

    Read the article

  • Analyze your IIS Log Files - Favorite Log Parser Queries

    - by The Official Microsoft IIS Site
    The other day I was asked if I knew about a tool that would allow users to easily analyze the IIS Log Files, to process and look for specific data that could easily be automated. My recommendation was that if they were comfortable with using a SQL-like language that they should use Log Parser . Log Parser is a very powerful tool that provides a generic SQL-like language on top of many types of data like IIS Logs, Event Viewer entries, XML files, CSV files, File System and others; and it allows you...(read more)

    Read the article

  • Analyze Your SEO Competition

    It is vital to analyze your Search Engine Optimization or SEO competition as part of running any business. With your online business technology has made it a relatively simple task, but it is essential to verify regularly where you stand against your competition.

    Read the article

  • Analyze your IIS Log Files - Favorite Log Parser Queries

    The other day I was asked if I knew about a tool that would allow users to easily analyze the IIS Log Files, to process and look for specific data that could easily be automated. My recommendation was that if they were comfortable with using a SQL-like language that they should use Log Parser. Log Parser is a very powerful tool that provides a generic SQL-like language on top of many types of data like IIS Logs, Event Viewer entries, XML files, CSV files, File System and others; and it allows you...Did you know that DotNetSlackers also publishes .net articles written by top known .net Authors? We already have over 80 articles in several categories including Silverlight. Take a look: here.

    Read the article

  • How to analyze data

    - by Subhash Dike
    We are working on an application that allows user to search/read some content in a particular domain. We wanted to add some capability in the app which can suggest user some content based on the usage pattern (analyze data based on frequency and relevance). Currently every time user search or read something we do store that information in backend database. We would like to use this data to present some additional content to user. Could someone explain what kind of tools will be required for such a job and any example? And what this concept is called, data analysis? data mining? business intelligence? or something else? Update: Sorry for being too broad, here is an example SQL Database (Just to give an idea, actual db is little different with normalization and stuff) Table: UserArticles Fields: UserName | ArticleId | ArticleTitle | DateVisited | ArticleCategory Table: CategoryArticles Fields: Category | Article Title | Author etc. One Category may have one more articles. One user may have read the same article multiple times (in this case we place additional entry in the user article table. Task: Use the information availabel in UserArticle table and rank categories in order which would be presented to user automatically in other part of application. Factors to be considered are frequency and recency. This might be possible through simple queries or may require specialized tools. Either way, the task is what mention above. I am not too sure which route to take, hence the question. Thoughts??

    Read the article

  • VSS Analyze - Access to file [filename] is denied

    - by AJ
    Our VSS database appears to be horribly out of shape. I've been trying to archive and run "analyze" and keep getting "Access to file [filename] is denied. The file may be read-only, may be in use, or you may not have permission to write to the file. Correct this problem and run analyze again." No one is logged into SourceSafe (including myself) and I'm running the analyze utility from the VS command prompt as follows: analyze -v -f -bbackuppath databasepath I get similar errors if I try and create project archives from the ssadmin tool. The database is on a network share, and we're running VSS 2005 v8.0.50727.42. I'd love to be able to do this, as it would be a first step in a move away from VSS. Thanks in advance. More Info Every time I run analyze, the file that spawns the access denied message changes. It's almost as if running analyze unlocks that file so that the next time I get through to the next one.

    Read the article

  • Analyze Drupal and Wordpress sites CPU load in shared server

    - by Tedi
    Our hosting company is complaining that both our Drupal and Wordpress websites running in a shared server are consuming too many CPU resources. The traffic for each site is not more than 100 users per day and, at a first glance, we don't have very many plugins/add-ons. Is there any tool or resource to analyse what is causing that high CPU load? Thanks Update: We decided to suspend our accounts while the problem was being debugged but still our hosting (Site5) said that they saw unacceptable activity on our sites so we had to move to a dedicated server... asked them several times to provide us with more information and they always came back saying that we had to purchase a higher account. Finally decided to move to another hosting service.

    Read the article

  • Online SEO Tool to Analyze Your Website

    Learning search engine optimization can be difficult without some sort of SEO tool to help decipher in which direction your marketing efforts are headed. Though there are, in fact, many online SEO tools that can help with your internet marketing campaign, perhaps the most beneficial is an SEO tool that analyzes your entire website. This type of tool, usually offered free of charge, acts in a similar way to search engine crawlers.

    Read the article

  • Detect, Analyze, Act – Fast!

    - by Ajay Khanna
    In fast changing business environment, it becomes crucial to identify business opportunities and business issues as soon as possible. If identified at the right time, business managers can address issues before they escalate to serious problems and can take advantage of the new opportunities before the competition does. Moreover, they have to be efficient to do this at the right cost. Success depends on how responsive organization is to emerging events and changing environment. These events can be customer issues, competition moves, changes in regulations, or changes in company policies. In order to be responsive in such situations, organizations need to first identify and track these situations. They can do that via business activity monitoring (BAM) and complex event processing (CEP). A unified monitoring dashboard helps put together a comprehensive picture of the situation in hand and provides deep insight to take proper actions. With CEP, businesses can connect all the relevant events, detect event patterns and take immediate actions using Business Process Management system.   So to be responsive we need: Real-Time Visibility with Business Activity Monitoring You can use BAM technology to monitor progress, track performance, meet service-level agreements (SLAs), manage exceptions, and issue alerts to an employee or application when a process is not functioning properly—all in real time. A unified monitoring dashboard helps you maintain a complete picture of each situation so you can take action effectively. BAM works hand in hand with BPM software to discover the significant activities that drive business success.   Real-Time Sense and Respond An event-driven BPM solution enables each step in a business process to be informed not only by the previous step, but also by any other step, data, and pattern of behavior deemed relevant to that step. This gives the company the ability to “sense and respond.” You can describe interesting event patterns and event correlations and monitor the business in real-time. Whenever a pre-defined pattern emerges you can take actions like raising alerts, notifications, or kicking off another business process. This synergy possible by integrating activity monitoring, event processing, and BPM makes it possible for managers to keep a finger on the pulse of their business. Business managers can now respond to customers faster, respond to competition faster, reduce fraud and do more cross-selling. Read more about being responsive in the whitepaper “The Instantly Responsive Enterprise: Integrating BPM and Complex Event Processing” in BPM Resource Kit.

    Read the article

  • Properly Analyze the Value of the Cloud

    Most analyses of the benefits of cloud computing are based on unrealistic total cost of ownership (TCO), return on investment (ROI), and capex/opex calculations. To fully understand the potential benefits of cloud computing, a new metric is required.

    Read the article

  • Help with Windows 7 BSOD with windbg minidump !analyze -v results

    - by Kurt Harless
    Hi gang, Windows 7 X64 Ultimate is BSODing occasionally. I suspect an overheating issue or something related to the use of my GTX-295 card that runs very hot. Here is an !analyze -v listing of the most recent minidump. Any and all help greatly appreciated. Kurt Microsoft (R) Windows Debugger Version 6.12.0002.633 AMD64 Copyright (c) Microsoft Corporation. All rights reserved. Loading Dump File [C:\Windows\Minidump\122810-31387-01.dmp] Mini Kernel Dump File: Only registers and stack trace are available Symbol search path is: SRV*c:\websymbols*http://msdl.microsoft.com/download/symbols Executable search path is: Windows 7 Kernel Version 7600 MP (8 procs) Free x64 Product: WinNt, suite: TerminalServer SingleUserTS Built by: 7600.16617.amd64fre.win7_gdr.100618-1621 Machine Name: Kernel base = 0xfffff800`03065000 PsLoadedModuleList = 0xfffff800`032a2e50 Debug session time: Tue Dec 28 11:04:03.597 2010 (UTC - 7:00) System Uptime: 2 days 2:28:40.407 Loading Kernel Symbols ............................................................... ................................................................ .............................................. Loading User Symbols Loading unloaded module list ................ ******************************************************************************* * * * Bugcheck Analysis * * * ******************************************************************************* Use !analyze -v to get detailed debugging information. BugCheck 3B, {c0000005, fffff800033b8873, fffff8800e322dc0, 0} Probably caused by : ntkrnlmp.exe ( nt!RtlCompareUnicodeStrings+c3 ) Followup: MachineOwner --------- 1: kd> !analyze -v ******************************************************************************* * * * Bugcheck Analysis * * * ******************************************************************************* SYSTEM_SERVICE_EXCEPTION (3b) An exception happened while executing a system service routine. Arguments: Arg1: 00000000c0000005, Exception code that caused the bugcheck Arg2: fffff800033b8873, Address of the instruction which caused the bugcheck Arg3: fffff8800e322dc0, Address of the context record for the exception that caused the bugcheck Arg4: 0000000000000000, zero. Debugging Details: ------------------ EXCEPTION_CODE: (NTSTATUS) 0xc0000005 - The instruction at 0x%08lx referenced memory at 0x%08lx. The memory could not be %s. FAULTING_IP: nt!RtlCompareUnicodeStrings+c3 fffff800`033b8873 488b7c2418 mov rdi,qword ptr [rsp+18h] CONTEXT: fffff8800e322dc0 -- (.cxr 0xfffff8800e322dc0) rax=0000000000000041 rbx=fffff8a015a3c1c0 rcx=0000000000000024 rdx=0000000000000003 rsi=fffff8800e3238b0 rdi=0000000000000009 rip=fffff800033b8873 rsp=fffff8800e323798 rbp=000000000000000d r8=fffff8a018cb374c r9=000000200a98fdc4 r10=fffff8800e323988 r11=fffff8800e32398e r12=fffff8a018127c18 r13=fffff8800126e550 r14=0000000000000001 r15=fffffa800abe1570 iopl=0 nv up ei pl nz ac po nc cs=0010 ss=0018 ds=002b es=002b fs=0053 gs=002b efl=00010216 nt!RtlCompareUnicodeStrings+0xc3: fffff800`033b8873 488b7c2418 mov rdi,qword ptr [rsp+18h] ss:0018:fffff880`0e3237b0=???????????????? Resetting default scope CUSTOMER_CRASH_COUNT: 1 DEFAULT_BUCKET_ID: VISTA_DRIVER_FAULT BUGCHECK_STR: 0x3B PROCESS_NAME: ccSvcHst.exe CURRENT_IRQL: 0 LAST_CONTROL_TRANSFER: from 0000000000000000 to fffff800033b8873 STACK_TEXT: fffff880`0e323798 00000000`00000000 : 00000000`00000000 00000000`00000000 00000000`00000000 00000000`00000000 : nt!RtlCompareUnicodeStrings+0xc3 FOLLOWUP_IP: nt!RtlCompareUnicodeStrings+c3 fffff800`033b8873 488b7c2418 mov rdi,qword ptr [rsp+18h] SYMBOL_STACK_INDEX: 0 SYMBOL_NAME: nt!RtlCompareUnicodeStrings+c3 FOLLOWUP_NAME: MachineOwner MODULE_NAME: nt IMAGE_NAME: ntkrnlmp.exe DEBUG_FLR_IMAGE_TIMESTAMP: 4c1c44a9 STACK_COMMAND: .cxr 0xfffff8800e322dc0 ; kb FAILURE_BUCKET_ID: X64_0x3B_nt!RtlCompareUnicodeStrings+c3 BUCKET_ID: X64_0x3B_nt!RtlCompareUnicodeStrings+c3 Followup: MachineOwner ---------

    Read the article

  • /analyze flag in Visual Studio 2010 Professional

    - by Martin
    Running Visual Studio 2008 Professional it is possible to enable static code analysis using the /analyze flag (even though this is not supported for the Professional version according to the documentation). In Visual Studio 2010 Professional this no longer works. Instead there is a default /analyze- flag added (one I can't find a GUI setting for). This does not work as well as the VS2008 version (or at all). Can anyone shed some light into this? What does the new /analyze- flag do and is there any way to enable the old analysis?

    Read the article

  • Linux kernel startup problems: how to analyze?

    - by java.is.for.desktop
    Hello, everyone! After manually updating the kernel from 2.6.33 to 2.6.34 on my OpenSuse 11.2 Notebook, it stops after the message Loading drivers, configuring devices... This stop can be interrupted with Ctrl-C, but when the system enters runlevel 5, no partitions are mounted (but the root partition), many services fail to start, and other strange things are going on. No X11. NOTE: I manually updated the kernel many times before, it worked. Yes, I know, in case of NVidia, the driver has to be recompiled. The question is: How can I analyze the cause of the problem? Doing dmesg gives me soooo much output, I can't "map" it to the output which I see at startup. The output does not contain the string Loading drivers, configuring devices, or similar.

    Read the article

  • build and analyze not working in my project

    - by mihirpmehta
    In my iPhone Project when i select build and analyze (shift + mac + A ) it will give me all potential memory leak in my project... but in my current project it is not working... when i intentionally put a memory leak and select build and analyze... it doesn't give me any potential memory leak as analyzer result please help...

    Read the article

  • Configure server on network to analyze traffic

    - by Strajan Sebastian
    I have the following network: http://i.stack.imgur.com/rapkH.jpg I want to send all the traffic from the devices that connect to the 192.168.0.1 router to the 192.168.10.1 router(and eventually to the Internet), by passing through the server and an additional router. Almost 2 days have passed and I can't figure what is wrong. While searching on the Internet for some similar configuration I found some articles that are somehow related to my needs, but the proposed solutions don't seem to work for me. This is a similar article: iptables forwarding between two interface I done the following steps for the configuration process: Set static IP address 192.168.1.90 for the eth0 on the server from the 192.168.1.1 router Set static IP address 192.168.0.90 for the eth1 on the server from the 192.168.0.1 router Forwarded all the traffic from 192.168.0.1 router to the server on eth1 interface witch seems to be working. The router firmware has some option to redirect all the traffic from all the ports to a specified address. Added the following rules on the server(Only the following, there aren't any additional rules): iptables -t nat -A POSTROUTING -o eth1 -j MASQUERADE iptables -A FORWARD -i eth1 -o eth0 -m state -–state RELATED,ESTABLISHED -j ACCEPT iptables -A FORWARD -i eth0 -o eth1 -j ACCEPT I also tried changing iptables -A FORWARD -i eth1 -o eth0 -m state -–state RELATED,ESTABLISHED -j ACCEPT into iptables -A FORWARD -i eth1 -o eth0 -j ACCEPT but still is not working. After adding the following to enable the packet forwarding for the server that is running CentOS: echo 1 /proc/sys/net/ipv4/ip_forward sysctl -w net.ipv4.ip_forward = 1 After a server restart and extra an extra check to see that all the configuration from above are still available I tried to see again if I can ping from a computer connected to 192.168.0.1/24 LAN the router from 192.168.1.1 but it didn't worked. The server has tshark(console wireshark) installed and I found that while sending a ping from a computer connected to 192.168.0.1 router to 192.168.1.1 the 192.168.0.90(eth1) receives the ping but it doesn't forward it to the eth0 interface as the rule tells: iptables -A FORWARD -i eth1 -o eth0 -j ACCEPT and don't now why this is happening. Questions: The iptables seem that don't work as I am expecting. Is there a need to add in the NAT table from iptables rules to redirect the traffic to the proper location, or is something else wrong with what I've done? I want to use tshark to view the traffic on the server because I think that is the best at doing this. Do you know something better that tshark to capture the traffic and maybe analyze it?

    Read the article

  • Please bear with me, can someone analyze this trace route please

    - by Abdulla
    Hello, my name is Abdulla and I'm from Kuwait. Sorry for my question as I know its not technically challenging. I'm facing some problems with my internet connection while gaming, I have DSL 2mb connection. My main problem is latency, in the morning its good but after that its gets really bad. My internet provider says there's nothing wrong and that everything is working perfectly. I tried to explain to them the latency issue but they say that as long as I'm getting the download speed there isn't anything I can do about it. I only want to know if this is true and that the company can't do anything before I change my internet provider, as I feel that the guys at the contact center might getting back to me without asking tech support. Below are 2 traces I made, one in the morning and the other in the afternoon: This was taken around 17:00 Microsoft Windows XP [Version 5.1.2600] (C) Copyright 1985-2001 Microsoft Corp. C:\Documents and Settings\Administrator>ping google.com Pinging google.com [66.102.9.104] with 32 bytes of data: Reply from 66.102.9.104: bytes=32 time=387ms TTL=49 Reply from 66.102.9.104: bytes=32 time=388ms TTL=49 Reply from 66.102.9.104: bytes=32 time=375ms TTL=49 Reply from 66.102.9.104: bytes=32 time=375ms TTL=49 Ping statistics for 66.102.9.104: Packets: Sent = 4, Received = 4, Lost = 0 (0% loss), Approximate round trip times in milli-seconds: Minimum = 375ms, Maximum = 388ms, Average = 381ms C:\Documents and Settings\Administrator>ping google.com /t Pinging google.com [66.102.9.104] with 32 bytes of data: Reply from 66.102.9.104: bytes=32 time=376ms TTL=49 Reply from 66.102.9.104: bytes=32 time=382ms TTL=49 Reply from 66.102.9.104: bytes=32 time=371ms TTL=49 Reply from 66.102.9.104: bytes=32 time=378ms TTL=49 Reply from 66.102.9.104: bytes=32 time=374ms TTL=49 Reply from 66.102.9.104: bytes=32 time=371ms TTL=49 Reply from 66.102.9.104: bytes=32 time=365ms TTL=49 Reply from 66.102.9.104: bytes=32 time=366ms TTL=49 Reply from 66.102.9.104: bytes=32 time=353ms TTL=49 Reply from 66.102.9.104: bytes=32 time=331ms TTL=49 Reply from 66.102.9.104: bytes=32 time=333ms TTL=49 Reply from 66.102.9.104: bytes=32 time=348ms TTL=49 Reply from 66.102.9.104: bytes=32 time=365ms TTL=49 Reply from 66.102.9.104: bytes=32 time=346ms TTL=49 Reply from 66.102.9.104: bytes=32 time=335ms TTL=49 Reply from 66.102.9.104: bytes=32 time=340ms TTL=49 Reply from 66.102.9.104: bytes=32 time=344ms TTL=49 Reply from 66.102.9.104: bytes=32 time=333ms TTL=49 Reply from 66.102.9.104: bytes=32 time=328ms TTL=49 Reply from 66.102.9.104: bytes=32 time=332ms TTL=49 Reply from 66.102.9.104: bytes=32 time=326ms TTL=49 Reply from 66.102.9.104: bytes=32 time=333ms TTL=49 Reply from 66.102.9.104: bytes=32 time=325ms TTL=49 Reply from 66.102.9.104: bytes=32 time=333ms TTL=49 Reply from 66.102.9.104: bytes=32 time=338ms TTL=49 Reply from 66.102.9.104: bytes=32 time=341ms TTL=49 Ping statistics for 66.102.9.104: Packets: Sent = 26, Received = 26, Lost = 0 (0% loss), Approximate round trip times in milli-seconds: Minimum = 325ms, Maximum = 382ms, Average = 348ms Control-C ^C C:\Documents and Settings\Administrator>travert google.com 'travert' is not recognized as an internal or external command, operable program or batch file. C:\Documents and Settings\Administrator>tracert google.com Tracing route to google.com [66.102.9.104] over a maximum of 30 hops: 1 <1 ms <1 ms <1 ms 192.168.0.1 2 6 ms 6 ms 6 ms 80-184-31-1.adsl.kems.net [80.184.31.1] 3 7 ms 7 ms 8 ms 168.187.0.226 4 7 ms 8 ms 9 ms 168.187.0.125 5 180 ms 187 ms 188 ms if-11-2.core1.RSD-Riyad.as6453.net [116.0.78.89] 6 209 ms 222 ms 204 ms 195.219.167.57 7 541 ms 536 ms 540 ms 195.219.167.42 8 553 ms 552 ms 538 ms Vlan1102.icore1.PVU-Paris.as6453.net [195.219.24 1.109] 9 547 ms 543 ms 542 ms xe-9-1-0.edge4.paris1.level3.net [4.68.110.213] 10 540 ms 523 ms 531 ms ae-33-51.ebr1.Paris1.Level3.net [4.69.139.193] 11 755 ms 761 ms 695 ms ae-45-45.ebr1.London1.Level3.net [4.69.143.101] 12 271 ms 263 ms 400 ms ae-11-51.car1.London1.Level3.net [4.69.139.66] 13 701 ms 730 ms 742 ms 195.50.118.210 14 659 ms 641 ms 660 ms 209.85.255.76 15 280 ms 283 ms 292 ms 209.85.251.190 16 308 ms 293 ms 296 ms 72.14.232.239 17 679 ms 700 ms 721 ms 64.233.174.18 18 268 ms 281 ms 269 ms lm-in-f104.1e100.net [66.102.9.104] Trace complete. C:\Documents and Settings\Administrator> This was taken at 10:00am Microsoft Windows XP [Version 5.1.2600] (C) Copyright 1985-2001 Microsoft Corp. C:\Documents and Settings\Administrator>ping google.com Pinging google.com [66.102.9.106] with 32 bytes of data: Reply from 66.102.9.106: bytes=32 time=110ms TTL=49 Reply from 66.102.9.106: bytes=32 time=111ms TTL=49 Reply from 66.102.9.106: bytes=32 time=112ms TTL=49 Reply from 66.102.9.106: bytes=32 time=120ms TTL=49 Ping statistics for 66.102.9.106: Packets: Sent = 4, Received = 4, Lost = 0 (0% loss), Approximate round trip times in milli-seconds: Minimum = 110ms, Maximum = 120ms, Average = 113ms C:\Documents and Settings\Administrator>ping google.com /t Pinging google.com [66.102.9.106] with 32 bytes of data: Reply from 66.102.9.106: bytes=32 time=109ms TTL=49 Reply from 66.102.9.106: bytes=32 time=110ms TTL=49 Reply from 66.102.9.106: bytes=32 time=111ms TTL=49 Reply from 66.102.9.106: bytes=32 time=111ms TTL=49 Reply from 66.102.9.106: bytes=32 time=112ms TTL=49 Reply from 66.102.9.106: bytes=32 time=112ms TTL=49 Reply from 66.102.9.106: bytes=32 time=116ms TTL=49 Reply from 66.102.9.106: bytes=32 time=110ms TTL=49 Reply from 66.102.9.106: bytes=32 time=109ms TTL=49 Reply from 66.102.9.106: bytes=32 time=110ms TTL=49 Reply from 66.102.9.106: bytes=32 time=109ms TTL=49 Reply from 66.102.9.106: bytes=32 time=110ms TTL=49 Reply from 66.102.9.106: bytes=32 time=112ms TTL=49 Reply from 66.102.9.106: bytes=32 time=109ms TTL=49 Reply from 66.102.9.106: bytes=32 time=110ms TTL=49 Reply from 66.102.9.106: bytes=32 time=115ms TTL=49 Reply from 66.102.9.106: bytes=32 time=110ms TTL=49 Reply from 66.102.9.106: bytes=32 time=109ms TTL=49 Reply from 66.102.9.106: bytes=32 time=110ms TTL=49 Reply from 66.102.9.106: bytes=32 time=113ms TTL=49 Reply from 66.102.9.106: bytes=32 time=115ms TTL=49 Reply from 66.102.9.106: bytes=32 time=109ms TTL=49 Reply from 66.102.9.106: bytes=32 time=110ms TTL=49 Ping statistics for 66.102.9.106: Packets: Sent = 32, Received = 32, Lost = 0 (0% loss), Approximate round trip times in milli-seconds: Minimum = 109ms, Maximum = 135ms, Average = 112ms Control-C ^C C:\Documents and Settings\Administrator>tracert google.com Tracing route to google.com [66.102.9.104] over a maximum of 30 hops: 1 <1 ms <1 ms <1 ms 192.168.0.1 2 6 ms 6 ms 6 ms 80-184-31-1.adsl.kems.net [80.184.31.1] 3 8 ms 7 ms 6 ms 168.187.0.226 4 6 ms 7 ms 7 ms 168.187.0.125 5 20 ms 20 ms 18 ms if-11-2.core1.RSD-Riyad.as6453.net [116.0.78.89] 6 171 ms 205 ms 215 ms 195.219.167.57 7 191 ms 215 ms 226 ms 195.219.167.42 8 * 103 ms 94 ms Vlan1102.icore1.PVU-Paris.as6453.net [195.219.24 1.109] 9 94 ms 95 ms 97 ms xe-9-1-0.edge4.paris1.level3.net [4.68.110.213] 10 94 ms 94 ms 94 ms ae-33-51.ebr1.Paris1.Level3.net [4.69.139.193] 11 101 ms 101 ms 101 ms ae-48-48.ebr1.London1.Level3.net [4.69.143.113] 12 102 ms 102 ms 101 ms ae-11-51.car1.London1.Level3.net [4.69.139.66] 13 103 ms 102 ms 103 ms 195.50.118.210 14 137 ms 103 ms 100 ms 209.85.255.76 15 130 ms 124 ms 124 ms 209.85.251.190 16 114 ms 116 ms 116 ms 72.14.232.239 17 135 ms 113 ms 126 ms 64.233.174.18 18 126 ms 125 ms 127 ms lm-in-f104.1e100.net [66.102.9.104] Trace complete. C:\Documents and Settings\Administrator>

    Read the article

  • Analyze a BSOD (irql_less_than_or_equal)

    - by Bruno Reis
    Hello. About 2 months ago I bought a new system and built it at home: Mother board: XFX X58i Processor: Core i7 920, using the stock cooler Memory: 3x2GB Corsair DDR3 1600 Video card: NVIDIA GTS 250 (1GB) Hard disk: 2x WD 500GB, 7200rpm I have 2 screens plugged into the video card, and the system is connected to a 550W PSU. Nothing is overclocked. After building the system, I stressed it a lot with Prime95 and rthdribl to check its stability. All my tests were perfect. So I reinstalled Win 7 x64 Professional and started using it normally. The first week (2010-03-15) I got the infamous irql_less_than_or_equal BSOD. Ten days after (2010-03-24) I got another one. Then on 2010-04-09, 2010-05-04. Since 2 days ago it became worse: I got one bluescreen per day! (2010-05-12, 2010-05-13, 2010-05-14). I installed BlueScreenView to try to obtain some information, but I'm not able to extract any useful information apart from the bug check string (irql_less_than_or_equal), and that it was caused by ntoskrnl.exe (the first three at ntoskrnl.exe+71f00, the last 4 at ntoskrnl.exe+70600 -- which I suspect could be the same thing, as Microsoft could have patched this file in the mean time, so the address of the function causing it changed). Then I stressed my memory sticks with memtest, they worked perfectly. After booting, I've stressed my GPU with FurMark and RTHDRIBL, everything was fine. Then I stressed the CPU with 4 instances of Prime95 while monitoring the temperature -- that never exceeded 85oC with the case closed --, everything fine. Finally I've stressed the whole system with HeavyLoad for a looooong time, everything worked just fine. So, I have stressed most of the components of the system, but couldn't get any useful information from it. Do you have any hint on what else can I do to find the culprit? Thanks Bruno

    Read the article

  • Analyze a wireless network that constantly drops/has speed issues

    - by Eddie Parker
    Hello, I'm curious what the best tools are to use for analyzing problems with a WiFi network. Here's the scenario: I have a WiFi router (Belkin N+) that's setup in AP mode. I have three RT-N13U's that I've purchased to use as 'repeaters', but I've had so many problems when more than one of them is running (bad routes) that I've only got one active. Sometimes certain boxes on my network can't talk to others, and drops are quite frequent and quite aggravating. I'm running Mac, Windows, and Linux (Gentoo) boxes on this network, so any software, or steps I should take that work for any of those boxes should be sufficient. Apologies if this is answered somewhere else - I'll close it as a dupe if so.

    Read the article

  • Highly robust and scalable search server needed for managing and analyze files

    - by ChrisBenyamin
    Hi everybody, I am looking for a professional search server system with functionality, like e.g. solr http://lucene.apache.org/solr/ holds. Place of action should be a centralized location, whereon many hosts would request data. Furthermore the system should be extensible for implementing statistical procedures. (e.g. a kind of heatmap (or common diagrams) of a (or more) file(s) (which has a guid), that is spread on different hosts.) This software doesn't have to be opensource. thanks. chris

    Read the article

1 2 3 4 5 6 7 8 9 10 11 12  | Next Page >