This project has retired. For details please refer to its Attic page.
GiraphStats xref
View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.giraph.counters;
20  
21  import org.apache.hadoop.mapreduce.Mapper.Context;
22  
23  import java.util.ArrayList;
24  import java.util.Arrays;
25  import java.util.Iterator;
26  import java.util.List;
27  
28  /**
29   * Hadoop Counters in group "Giraph Stats". General statistics about job.
30   */
31  public class GiraphStats extends HadoopCountersBase {
32    /** Counter group name for the Giraph statistics */
33    public static final String GROUP_NAME = "Giraph Stats";
34    /** superstep counter name */
35    public static final String SUPERSTEP_NAME = "Superstep";
36    /** aggregate vertices counter name */
37    public static final String VERTICES_NAME = "Aggregate vertices";
38    /** aggregate finished vertices counter name */
39    public static final String FINISHED_VERTICES_NAME =
40        "Aggregate finished vertices";
41    /** aggregate edges counter name */
42    public static final String EDGES_NAME = "Aggregate edges";
43    /** sent messages counter name */
44    public static final String SENT_MESSAGES_NAME = "Sent messages";
45    /** sent message bytes counter name */
46    public static final String SENT_MESSAGE_BYTES_NAME = "Sent message bytes";
47    /** aggregate sent messages counter name */
48    public static final String AGGREGATE_SENT_MESSAGES_NAME
49      = "Aggregate sent messages";
50    /** aggregate sent messages bytes counter name */
51    public static final String AGGREGATE_SENT_MESSAGE_BYTES_NAME
52      = "Aggregate sent message bytes";
53    /** workers counter name */
54    public static final String CURRENT_WORKERS_NAME = "Current workers";
55    /** current master partition task counter name */
56    public static final String CURRENT_MASTER_PARTITION_TASK_NAME =
57        "Current master task partition";
58    /** last checkpointed superstep counter name */
59    public static final String LAST_CHECKPOINTED_SUPERSTEP_NAME =
60        "Last checkpointed superstep";
61    /** aggregate bytes loaded from local disks in out-of-core */
62    public static final String OOC_BYTES_LOADED_NAME =
63        "Aggregate bytes loaded from local disks (out-of-core)";
64    /** aggregate bytes stored to local disks in out-of-core */
65    public static final String OOC_BYTES_STORED_NAME =
66        "Aggregate bytes stored to local disks (out-of-core)";
67    /** lowest percentage of graph in memory throughout the execution */
68    public static final String LOWEST_GRAPH_PERCENTAGE_IN_MEMORY_NAME =
69        "Lowest percentage of graph in memory so far (out-of-core)";
70  
71    /** Singleton instance for everyone to use */
72    private static GiraphStats INSTANCE;
73  
74    /** Superstep counter */
75    private static final int SUPERSTEP = 0;
76    /** Vertex counter */
77    private static final int VERTICES = 1;
78    /** Finished vertex counter */
79    private static final int FINISHED_VERTICES = 2;
80    /** Edge counter */
81    private static final int EDGES = 3;
82    /** Sent messages counter */
83    private static final int SENT_MESSAGES = 4;
84    /** Workers on this superstep */
85    private static final int CURRENT_WORKERS = 5;
86    /** Current master task partition */
87    private static final int CURRENT_MASTER_TASK_PARTITION = 6;
88    /** Last checkpointed superstep */
89    private static final int LAST_CHECKPOINTED_SUPERSTEP = 7;
90    /** Sent message bytes counter */
91    private static final int SENT_MESSAGE_BYTES = 8;
92    /** Aggregate sent messages counter */
93    private static final int AGG_SENT_MESSAGES = 9;
94    /** Aggregate sent message bytes counter */
95    private static final int AGG_SENT_MESSAGE_BYTES = 10;
96    /** Aggregate OOC loaded bytes counter */
97    private static final int OOC_BYTES_LOADED = 11;
98    /** Aggregate OOC stored bytes counter */
99    private static final int OOC_BYTES_STORED = 12;
100   /** Lowest percentage of graph in memory over time */
101   private static final int LOWEST_GRAPH_PERCENTAGE_IN_MEMORY = 13;
102   /** Number of counters in this class */
103   private static final int NUM_COUNTERS = 14;
104 
105   /** All the counters stored */
106   private final GiraphHadoopCounter[] counters;
107 
108   /**
109    * Create with Hadoop Context.
110    *
111    * @param context Hadoop Context to use.
112    */
113   private GiraphStats(Context context) {
114     super(context, GROUP_NAME);
115     counters = new GiraphHadoopCounter[NUM_COUNTERS];
116     counters[SUPERSTEP] = getCounter(SUPERSTEP_NAME);
117     counters[VERTICES] = getCounter(VERTICES_NAME);
118     counters[FINISHED_VERTICES] = getCounter(FINISHED_VERTICES_NAME);
119     counters[EDGES] = getCounter(EDGES_NAME);
120     counters[SENT_MESSAGES] = getCounter(SENT_MESSAGES_NAME);
121     counters[SENT_MESSAGE_BYTES] = getCounter(SENT_MESSAGE_BYTES_NAME);
122     counters[CURRENT_WORKERS] = getCounter(CURRENT_WORKERS_NAME);
123     counters[CURRENT_MASTER_TASK_PARTITION] =
124         getCounter(CURRENT_MASTER_PARTITION_TASK_NAME);
125     counters[LAST_CHECKPOINTED_SUPERSTEP] =
126         getCounter(LAST_CHECKPOINTED_SUPERSTEP_NAME);
127     counters[AGG_SENT_MESSAGES] =
128         getCounter(AGGREGATE_SENT_MESSAGES_NAME);
129     counters[AGG_SENT_MESSAGE_BYTES] =
130         getCounter(AGGREGATE_SENT_MESSAGE_BYTES_NAME);
131     counters[OOC_BYTES_LOADED] = getCounter(OOC_BYTES_LOADED_NAME);
132     counters[OOC_BYTES_STORED] = getCounter(OOC_BYTES_STORED_NAME);
133     counters[LOWEST_GRAPH_PERCENTAGE_IN_MEMORY] =
134         getCounter(LOWEST_GRAPH_PERCENTAGE_IN_MEMORY_NAME);
135     counters[LOWEST_GRAPH_PERCENTAGE_IN_MEMORY].setValue(100);
136   }
137 
138   /**
139    * Initialize with Hadoop Context.
140    *
141    * @param context Hadoop Context to use.
142    */
143   public static void init(Context context) {
144     INSTANCE = new GiraphStats(context);
145   }
146 
147   /**
148    * Get singleton instance.
149    *
150    * @return GiraphStats singleton
151    */
152   public static GiraphStats getInstance() {
153     return INSTANCE;
154   }
155 
156 
157   /**
158    * Get a map of counter names, and values
159    * To be used by the master to send to the job client
160    *
161    * @return Map of counter names and values
162    */
163   public List<CustomCounter> getCounterList() {
164     List<CustomCounter> counterList = new ArrayList<>();
165     for (int i = 0; i < counters.length; i++) {
166       counterList.add(new CustomCounter(GROUP_NAME,
167               counters[i].getDisplayName(),
168               CustomCounter.Aggregation.SUM, counters[i].getValue()));
169     }
170     return counterList;
171   }
172 
173   /**
174    * Get SuperstepCounter counter
175    *
176    * @return SuperstepCounter counter
177    */
178   public GiraphHadoopCounter getSuperstepCounter() {
179     return counters[SUPERSTEP];
180   }
181 
182   /**
183    * Get Vertices counter
184    *
185    * @return Vertices counter
186    */
187   public GiraphHadoopCounter getVertices() {
188     return counters[VERTICES];
189   }
190 
191   /**
192    * Get FinishedVertexes counter
193    *
194    * @return FinishedVertexes counter
195    */
196   public GiraphHadoopCounter getFinishedVertexes() {
197     return counters[FINISHED_VERTICES];
198   }
199 
200   /**
201    * Get Edges counter
202    *
203    * @return Edges counter
204    */
205   public GiraphHadoopCounter getEdges() {
206     return counters[EDGES];
207   }
208 
209   /**
210    * Get SentMessages counter
211    *
212    * @return SentMessages counter
213    */
214   public GiraphHadoopCounter getSentMessages() {
215     return counters[SENT_MESSAGES];
216   }
217 
218   /**
219    * Get SentMessageBytes counter
220    *
221    * @return SentMessageBytes counter
222    */
223   public GiraphHadoopCounter getSentMessageBytes() {
224     return counters[SENT_MESSAGE_BYTES];
225   }
226 
227   /**
228    * Get AggregateSentMessages counter
229    *
230    * @return AggregateSentMessages counter
231    */
232   public GiraphHadoopCounter getAggregateSentMessages() {
233     return counters[AGG_SENT_MESSAGES];
234   }
235 
236   /**
237    * Get AggregateSentMessageBytes counter
238    *
239    * @return AggregateSentMessageBytes counter
240    */
241   public GiraphHadoopCounter getAggregateSentMessageBytes() {
242     return counters[AGG_SENT_MESSAGE_BYTES];
243   }
244 
245   /**
246    * Get CurrentWorkers counter
247    *
248    * @return CurrentWorkers counter
249    */
250   public GiraphHadoopCounter getCurrentWorkers() {
251     return counters[CURRENT_WORKERS];
252   }
253 
254   /**
255    * Get CurrentMasterTaskPartition counter
256    *
257    * @return CurrentMasterTaskPartition counter
258    */
259   public GiraphHadoopCounter getCurrentMasterTaskPartition() {
260     return counters[CURRENT_MASTER_TASK_PARTITION];
261   }
262 
263   /**
264    * Get LastCheckpointedSuperstep counter
265    *
266    * @return LastCheckpointedSuperstep counter
267    */
268   public GiraphHadoopCounter getLastCheckpointedSuperstep() {
269     return counters[LAST_CHECKPOINTED_SUPERSTEP];
270   }
271 
272   /**
273    * Get OOCBytesLoaded counter
274    *
275    * @return OOCBytesLoaded counter
276    */
277   public GiraphHadoopCounter getAggregateOOCBytesLoaded() {
278     return counters[OOC_BYTES_LOADED];
279   }
280 
281   /**
282    * Get OOCBytesStored counter
283    *
284    * @return OOCBytesStored counter
285    */
286   public GiraphHadoopCounter getAggregateOOCBytesStored() {
287     return counters[OOC_BYTES_STORED];
288   }
289 
290   public GiraphHadoopCounter getLowestGraphPercentageInMemory() {
291     return counters[LOWEST_GRAPH_PERCENTAGE_IN_MEMORY];
292   }
293 
294   @Override
295   public Iterator<GiraphHadoopCounter> iterator() {
296     return Arrays.asList(counters).iterator();
297   }
298 }