Add option to only load entities from chunk and use it for genPOI, resulting in roughly 1/3 runtime reduction.

.travis.yml

@@ -4,10 +4,10 @@ python:
   - "2.7"
   # - "3.2"
 env:
-  - MC_VERSION=1.8
+  - MC_VERSION=1.9
 before_install:
-  - wget http://hg.effbot.org/pil-117/raw/f356a1f64271/libImaging/Imaging.h
-  - wget http://hg.effbot.org/pil-117/raw/f356a1f64271/libImaging/ImPlatform.h
+  - wget https://raw.githubusercontent.com/python-pillow/Pillow/master/libImaging/Imaging.h
+  - wget https://raw.githubusercontent.com/python-pillow/Pillow/master/libImaging/ImPlatform.h
 install:
   - pip install -q pillow
   - pip install -q numpy
@@ -22,8 +22,7 @@ notifications:
   email: false
   irc:
     channels:
-      - "irc.freenode.org#overviewer"
-    skip_join: true
+      - "chat.freenode.net#overviewer"
     template:
       - "\x0313Minecraft-Overviewer\x03/\x0306%{branch}\x03 \x0314%{commit}\x03 %{build_url} %{message}"
 # matrix:

MANIFEST.in

@@ -4,10 +4,10 @@ include CONTRIBUTORS.rst
 include overviewer.py
 include contribManager.py
 include sample_config.py
-recursive-include overviewer_core/ *.py
-recursive-include overviewer_core/src/ *.c *.h
-recursive-include overviewer_core/src/primitives/ *.c *.h
-recursive-include overviewer_core/data/ *
-recursive-include contrib/ *.py
-recursive-include docs/ *
+recursive-include overviewer_core *.py
+recursive-include overviewer_core/src *.c *.h
+recursive-include overviewer_core/src/primitives *.c *.h
+recursive-include overviewer_core/data *
+recursive-include contrib *.py
+recursive-include docs *
 prune docs/_build

README.rst

@@ -85,4 +85,4 @@ https://github.com/overviewer/Minecraft-Overviewer/issues
 Feel free to comment on issues, report new issues, and vote on issues that are
 important to you.
 
-.. |Build Status| image:: https://secure.travis-ci.org/overviewer/Minecraft-Overviewer.png?branch=master
+.. |Build Status| image:: https://secure.travis-ci.org/overviewer/Minecraft-Overviewer.svg?branch=master

docs/building.rst

@@ -72,6 +72,36 @@ then try the following::
 
 If the build was successful, there should be a c_overviewer.pyd file in your current working directory.
 
+Building with mingw-w64 and msys2
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is the recommended way to build on Windows without MSVC.
+
+1. Install msys2 by following **all** the instructions on 
+   `the msys2 installation page <https://msys2.github.io/>`_.
+
+2. Install the dependencies::
+
+    pacman -S git mingw-w64-x86_64-python2-numpy mingw-w64-x86_64-python2-Pillow mingw-w64-x86_64-python2 mingw-w64-x86_64-toolchain
+
+3. Clone the Minecraft-Overviewer git repository::
+
+    git clone https://github.com/overviewer/Minecraft-Overviewer.git
+
+   The source code will be downloaded to your msys2 home directory, e.g.
+   ``C:\msys2\home\Potato\Minecraft-Overviewer``
+
+4. Close the msys2 shell. Instead, open the MinGW64 shell.
+
+5. Build the Overviewer by changing your current working directory into the source
+   directory and executing the build script::
+
+    cd Minecraft-Overviewer
+    python2 setup.py build
+
+After it finishes, you should now be able to execute ``overviewer.py`` from the MINGW64
+shell.
+
 Building with mingw
 ~~~~~~~~~~~~~~~~~~~
 

docs/config.rst

@@ -1172,13 +1172,13 @@ StructureOverlay
 
     Example::
 
-        MineralOverlay(minerals=[(((0, 0, 0, 66), (0, -1, 0, 4)), (255, 0, 0, 255)),
-                                 (((0, 0, 0, 27), (0, -1, 0, 4)), (0, 255, 0, 255))])
+        StructureOverlay(structures=[(((0, 0, 0, 66), (0, -1, 0, 4)), (255, 0, 0, 255)),
+                                     (((0, 0, 0, 27), (0, -1, 0, 4)), (0, 255, 0, 255))])
 
     In this example all rails(66) on top of cobblestone are rendered in
     pure red. And all powerrails(27) are rendered in green.
 
-    If ``minerals`` is not provided, a default rail coloring is used.
+    If ``structures`` is not provided, a default rail coloring is used.
 
 BiomeOverlay
     Color the map according to the biome at that point. Either use on

docs/signs.rst

@@ -220,6 +220,9 @@ POI markers. For example::
 .. note::
     A --genpoi run will NOT generate a map render, it will only generate markers.
 
+If all went well, you will see a "Markers" button in the upper-right corner of
+your map.
+
 genPOI.py
 ---------
 

overviewer_core/aux_files/genPOI.py

@@ -24,14 +24,17 @@
 import sys
 import time
 import urllib2
+import datetime
 
 from collections import defaultdict
+from contextlib import closing
 from multiprocessing import Pool
 from optparse import OptionParser
 
 from overviewer_core import logger
 from overviewer_core import nbt
 from overviewer_core import configParser, world
+from overviewer_core.files import FileReplacer
 
 UUID_LOOKUP_URL = 'https://sessionserver.mojang.com/session/minecraft/profile/'
 
@@ -161,7 +164,7 @@ def handleEntities(rset, config, config_path, filters, markers):
     if numbuckets == 1:
         for (x, z, mtime) in rset.iterate_chunks():
             try:
-                data = rset.get_chunk(x, z)
+                data = rset.get_chunk(x, z, entities_only=True)
                 for poi in itertools.chain(data['TileEntities'], data['Entities']):
                     if poi['id'] == 'Sign': # kill me
                         poi = signWrangler(poi)
@@ -210,19 +213,35 @@ class PlayerDict(dict):
     def load_cache(cls, outputdir):
         cache_file = os.path.join(outputdir, "uuidcache.dat")
         if os.path.exists(cache_file):
-            gz = gzip.GzipFile(cache_file) 
-            cls.uuid_cache = json.load(gz)
-            logging.info("Loaded UUID cache from %r with %d entries", cache_file, len(cls.uuid_cache.keys()))
+            try:
+                with closing(gzip.GzipFile(cache_file)) as gz:
+                    cls.uuid_cache = json.load(gz)
+                    logging.info("Loaded UUID cache from %r with %d entries",
+                                 cache_file, len(cls.uuid_cache.keys()))
+            except (ValueError, IOError):
+                logging.warning("Failed to load UUID cache -- it might be corrupt")
+                cls.uuid_cache = {}
+                corrupted_cache = cache_file + ".corrupted." + datetime.datetime.now().isoformat()
+                try:
+                    os.rename(cache_file, corrupted_cache)
+                    logging.warning("If %s does not appear to contain meaningful data, you may safely delete it", corrupted_cache)
+                except OSError:
+                    logging.warning("Failed to backup corrupted UUID cache")
+
+                logging.info("Initialized an empty UUID cache")
         else:
             cls.uuid_cache = {}
             logging.info("Initialized an empty UUID cache")
 
     @classmethod
     def save_cache(cls, outputdir):
         cache_file = os.path.join(outputdir, "uuidcache.dat")
-        gz = gzip.GzipFile(cache_file, "wb")
-        json.dump(cls.uuid_cache, gz)
-        logging.info("Wrote UUID cache with %d entries", len(cls.uuid_cache.keys()))
+
+        with FileReplacer(cache_file) as cache_file_name:
+            with closing(gzip.GzipFile(cache_file_name, "wb")) as gz:
+                json.dump(cls.uuid_cache, gz)
+                logging.info("Wrote UUID cache with %d entries",
+                             len(cls.uuid_cache.keys()))
 
     def __getitem__(self, item):
         if item == "EntityId":
@@ -280,7 +299,7 @@ def handlePlayers(worldpath, filters, markers):
             data.use_uuid = useUUIDs
             if isSinglePlayer:
                 data = data['Data']['Player']
-        except IOError:
+        except (IOError, TypeError):
             logging.warning("Skipping bad player dat file %r", playerfile)
             continue
 
@@ -513,7 +532,8 @@ def main():
     logging.info("Done handling POIs")
     logging.info("Writing out javascript files")
 
-    PlayerDict.save_cache(destdir)
+    if not options.skipplayers:
+        PlayerDict.save_cache(destdir)
 
     with open(os.path.join(destdir, "markersDB.js"), "w") as output:
         output.write("var markersDB=")

overviewer_core/data/web_assets/index.html

@@ -10,7 +10,7 @@
 
 <link rel="stylesheet" href="overviewer.css" type="text/css" />
 
-<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js"></script>
+<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script>
 <script type="text/javascript" src="https://maps.google.com/maps/api/js?sensor=false"></script>
 
 <script type="text/javascript" src="underscore.js"></script>

overviewer_core/observer.py

@@ -234,7 +234,7 @@ def __init__(self, outputdir, minrefresh=5, messages=False):
         else:
             raise Exception("JSObserver: messages parameter must be a dictionary with three entries: totalTiles, renderCompleted and renderProgress")
         if not os.path.exists(outputdir):
-            raise Exception("JSObserver: Output directory specified (%s) doesn't appear to exist. This should be the same as the Overviewer output directory")
+            raise Exception("JSObserver: Output directory specified (%s) doesn't appear to exist. This should be the same as the Overviewer output directory" % outputdir)
 
         self.logfile = open(os.path.join(outputdir, "progress.json"), "w+", 0)
         self.json["message"]="Render starting..."

overviewer_core/rcon.py

@@ -68,7 +68,13 @@ def send(self, t, payload):
         if not toread:
             raise RConException(self.rid, "Request timed out.")
 
-        res_len, res_id, res_type = struct.unpack("<iii", self.sock.recv(12))
+        try:
+            res_len, res_id, res_type = \
+                struct.unpack("<iii", self.sock.recv(12, socket.MSG_WAITALL))
+        except Exception as e:
+            raise RConException(self.rid,
+                                "RCon protocol error. Are you sure you're " + \
+                                "talking to the RCon port? Error: %s" % e)
         res_data = self.sock.recv(res_len - 4 - 4)
         res_data = res_data[:-2]
 

overviewer_core/src/iterate.c

@@ -263,6 +263,7 @@ is_stairs(int block) {
         case 163: /* acacia wood stairs */
         case 164: /* dark wood stairs */
         case 180: /* red sandstone stairs */
+        case 203: /* purpur stairs */
             return 1;
     }
     return 0;

overviewer_core/src/overviewer.h

@@ -26,12 +26,14 @@
 
 // increment this value if you've made a change to the c extesion
 // and want to force users to rebuild
-#define OVERVIEWER_EXTENSION_VERSION 49
+#define OVERVIEWER_EXTENSION_VERSION 50
 
 /* Python PIL, and numpy headers */
 #include <Python.h>
 #include <numpy/arrayobject.h>
 #include <Imaging.h>
+/* Fix Pillow on mingw-w64 which includes windows.h in Imaging.h */
+#undef TRANSPARENT
 
 /* like (a * b + 127) / 255), but much faster on most platforms
    from PIL's _imaging.c */

overviewer_core/src/primitives/lighting.c

@@ -95,7 +95,7 @@ calculate_light_color_fancy_night(void *data,
  * may (and probably should) pass NULL.
  */
 
-inline unsigned char
+unsigned char
 estimate_blocklevel(RenderPrimitiveLighting *self, RenderState *state,
                          int x, int y, int z, int *authoratative) {
 
@@ -138,7 +138,7 @@ estimate_blocklevel(RenderPrimitiveLighting *self, RenderState *state,
     blocklevel = get_data(state, BLOCKLIGHT, x, y, z);
 
     /* no longer a guess */
-    if (!(block == 44 || block == 53 || block == 67 || block == 108 || block == 109 || block == 180 || block == 182) && authoratative) {
+    if (!(block == 44 || block == 53 || block == 67 || block == 108 || block == 109 || block == 180 || block == 182 || block == 205) && authoratative) {
         *authoratative = 1;
     }
 
@@ -160,7 +160,8 @@ get_lighting_color(RenderPrimitiveLighting *self, RenderState *state,
     /* special half-step handling, stairs handling */
     /* Anvil also needs to be here, blockid 145 */
     if (block == 44 || block == 53 || block == 67 || block == 108 || block == 109 || block == 114 ||
-        block == 128 || block == 134 || block == 135 || block == 136 || block == 145 || block == 156 || block == 163 || block == 164 || block == 180 || block == 182) {
+        block == 128 || block == 134 || block == 135 || block == 136 || block == 145 || block == 156 ||
+        block == 163 || block == 164 || block == 180 || block == 182 || block == 203 || block == 205) {
         unsigned int upper_block;
 
         /* stairs and half-blocks take the skylevel from the upper block if it's transparent */
@@ -171,7 +172,8 @@ get_lighting_color(RenderPrimitiveLighting *self, RenderState *state,
             upper_block = get_data(state, BLOCKS, x, y + upper_counter, z);
         } while (upper_block == 44 || upper_block == 53 || upper_block == 67 || upper_block == 108 ||
                  upper_block == 109 || upper_block == 114 || upper_block == 128 || upper_block == 134 ||
-                 upper_block == 135 || upper_block == 136 || upper_block == 156 || upper_block == 163 || upper_block == 164 || upper_block == 180 || upper_block == 182);
+                 upper_block == 135 || upper_block == 136 || upper_block == 156 || upper_block == 163 ||
+                 upper_block == 164 || upper_block == 180 || upper_block == 182 || upper_block == 203 || upper_block == 205);
         if (is_transparent(upper_block)) {
             skylevel = get_data(state, SKYLIGHT, x, y + upper_counter, z);
         } else {