Merge 'develop' for gtools-1.0.1 (2018-07-23); bug fixes, typos

Bug fixes

* `gegen, replace` gave the wrong result if the source was also the target
  (`tag` and `group` worked, but all other stats did not).
* `gunique, by()` was affected by this bug and now works correctly.
* `gcollapse, merge` exits with error if there are no observations.
* Fixed typos in gtools.ado and help file

Enhancements

* `gunique, by()` no longer prints the top unique levels by default unless
  the user asks for them via `detail`.
* Specified gtop is an alias in readthedocs examples.
* Minor cosmetic updates to gtop

README.md

@@ -13,7 +13,7 @@ implementation of collapse, pctile, xtile, contract, egen, isid,
 levelsof, duplicates, and unique/distinct using C plugins for a massive
 speed improvement.
 
-`version 1.0.0 22Jul2018`
+`version 1.0.1 23Jul2018`
 Builds: Linux, OSX [![Travis Build Status](https://travis-ci.org/mcaceresb/stata-gtools.svg?branch=master)](https://travis-ci.org/mcaceresb/stata-gtools),
 Windows (Cygwin) [![Appveyor Build status](https://ci.appveyor.com/api/projects/status/2bh1q9bulx3pl81p/branch/master?svg=true)](https://ci.appveyor.com/project/mcaceresb/stata-gtools)
 

build.py

@@ -137,11 +137,11 @@ def makedirs_safe(directory):
 gtools_zip = [
     "changelog.md",
     "gtools.pkg",
-    "stata.toc"
+    "stata.toc",
 ] + gtools_ssc
 
 gtools_build = gtools_zip + [
-    "tests.do"
+    "gtools_tests.do"
 ]
 
 # ---------------------------------------------------------------------
@@ -380,7 +380,7 @@ def makedirs_safe(directory):
             f.write(linesep)
             f.write("cap net uninstall gtools")
             f.write(linesep)
-            f.write("net install gtools, from($builddir)")
+            f.write("net install gtools, from($builddir) replace")
             f.write(linesep)
 
         chdir(statadir)

build/_gtools_internal.ado

@@ -1,4 +1,4 @@
-*! version 1.0.0 21Jul2018 Mauricio Caceres Bravo, [email protected]
+*! version 1.0.1 23Jul2018 Mauricio Caceres Bravo, [email protected]
 *! Encode varlist using Jenkin's 128-bit spookyhash via C plugins
 
 * rc 17000
@@ -14,7 +14,7 @@
 
 capture program drop _gtools_internal
 program _gtools_internal, rclass
-    version 13
+    version 13.1
     global GTOOLS_USER_INTERNAL_VARABBREV `c(varabbrev)'
     * set varabbrev off
 
@@ -1385,7 +1385,7 @@ program _gtools_internal, rclass
         scalar __gtools_used_io   = 0
         scalar __gtools_ixfinish  = 0
         scalar __gtools_J         = _N
-        scalar __gtools_init_targ = ( "`ifin'" != "" ) & ("`merge'" != "")
+        scalar __gtools_init_targ = ("`ifin'" != "") & ("`merge'" != "")
 
         if inlist("`anything'", "forceio", "switch") {
             local extravars `__gtools_sources' `__gtools_sources' `freq'
@@ -1481,9 +1481,16 @@ program _gtools_internal, rclass
         *     2. Sort indexed hash
         *     3. Determine group sizes and cut points
         *     4. Use index and group info to compute the function
+        *
+        * NOTE: If there are targets (as with egen, collapse, or generic
+        * hash), they are replaced with missing values internally right
+        * before writing the output. Special functions tag, group,
+        * and count are initialized as well, should they have been
+        * requested.
 
-        if ( inlist("`gfunction'", "unique", "egen") ) {
+        if ( inlist("`gfunction'", "unique", "egen", "hash") ) {
             local gcall hash
+            scalar __gtools_init_targ = ("`ifin'" != "") & ("`replace'" != "")
         }
         else if ( inlist("`gfunction'",  "contract") ) {
             local 0 `gcontract'
@@ -2722,6 +2729,7 @@ program parse_by_types, rclass
             exit 17005
         }
         cap scalar drop __gtools_k_strL
+        * disp as txt "(note: performance with strL variables is not optimized)"
     }
     else if ( ("`varstrL'" != "") & ("`forcestrl'" == "") ) {
         disp as err _n(1) "gtools failed to parse strL variables."

build/changelog.md

@@ -1,6 +1,20 @@
 Change Log
 ==========
 
+## gtools-1.0.1 (2018-07-23)
+
+### Bug fixes
+
+* `gegen, replace` gave the wrong result if the source was also the target
+  (`tag` and `group` worked, but all other stats did not).
+* `gunique, by()` was affected by this bug and now works correctly.
+* `gcollapse, merge` exits with error if there are no observations matching `if' `in'.
+
+### Enhancements
+
+* `gunique, by()` no longer prints the top unique levels by default unless
+  the user asks for them via `detail`.
+
 ## gtools-1.0.0 (2018-07-21)
 
 First official release! This will go on SSC once all the tests have
@@ -11,7 +25,7 @@ passed. Final TODOs:
     [X] Windows with Stata 13
     [X] Windows with Stata 15
     [X] Linux with Stata 15
-    [ ] OSX with Stata 14
+    [X] OSX with Stata 14
 [X] Read the documentation (Stata)
 [X] Read the documentation (online)
 

build/fasterxtile.ado

@@ -4,7 +4,7 @@
 
 capture program drop fasterxtile
 program define fasterxtile
-    version 13
+    version 13.1
 
     if ( `=_N < 1' ) {
         error 2000

build/gcollapse.ado

@@ -1,9 +1,9 @@
-*! version 1.0.0 21Jul2018 Mauricio Caceres Bravo, [email protected]
+*! version 1.0.1 23Jul2018 Mauricio Caceres Bravo, [email protected]
 *! -collapse- implementation using C for faster processing
 
 capture program drop gcollapse
 program gcollapse, rclass
-    version 13
+    version 13.1
     global GTOOLS_USER_VARABBREV `c(varabbrev)'
     local 00 `0'
 
@@ -627,6 +627,11 @@ program gcollapse, rclass
         collapse `anything' `if' `in', `by' `cw' `fast'
         exit 0
     }
+    else if ( _rc == 17001 ) {
+        local rc = _rc
+        CleanExit
+        error 2000
+    }
     else if ( _rc ) {
         local rc = _rc
         CleanExit

build/gcollapse.sthlp

@@ -1,5 +1,5 @@
 {smcl}
-{* *! version 1.0.0 21Jul2018}{...}
+{* *! version 1.0.1 23Jul2018}{...}
 {viewerdialog gcollapse "dialog gcollapse"}{...}
 {vieweralsosee "[R] gcollapse" "mansection R gcollapse"}{...}
 {viewerjumpto "Syntax" "gcollapse##syntax"}{...}

build/gcontract.ado

@@ -3,7 +3,7 @@
 
 cap program drop gcontract
 program gcontract, rclass
-    version 13
+    version 13.1
 
     if ( `=_N' == 0 ) {
         di as err "no observations"

build/gdistinct.ado

@@ -3,7 +3,7 @@
 
 capture program drop gdistinct
 program gdistinct, rclass
-    version 13
+    version 13.1
 
     if ( `=_N < 1' ) {
         di as err "no observations"

build/gduplicates.ado

@@ -3,7 +3,7 @@
 
 capture program drop gduplicates
 program gduplicates, rclass
-    version 13
+    version 13.1
 
     local 00 `0'
     gettoken cmd 0 : 0, parse(" ,")

build/gegen.ado

@@ -1,4 +1,4 @@
-*! version 1.0.0 21Jul2018 Mauricio Caceres Bravo, [email protected]
+*! version 1.0.1 23Jul2018 Mauricio Caceres Bravo, [email protected]
 *! implementation -egen- using C for faster processing
 
 /*
@@ -23,7 +23,7 @@
 
 capture program drop gegen
 program define gegen, byable(onecall) rclass
-    version 13
+    version 13.1
 
     local 00 `0'
     qui syntax anything(equalok) [if] [in] [aw fw iw pw], [by(str) *]
@@ -287,20 +287,41 @@ program define gegen, byable(onecall) rclass
         tempvar dummy
         local rename rename `dummy' `name'
         local addvar qui mata: st_addvar("`type'", "`dummy'")
+        local noobs  ""
         local retype = `retype' & 1
     }
     else {
+
+        * NOTE: Addvar should be "" with replace; the problem was that
+        * the internals did not empty the variable before writing to
+        * it. With if/in conditions, this caused problems because the
+        * variable was not set to missing outside the range, as it
+        * should.
+        *
+        * As a quickfix I thought I could just empty it before calling
+        * internals. However, this causesd two issues: The variable
+        * would be missing on error, and if the target is also a source,
+        * the source would be all misssing when read by the plugin!
+        *
+        * The easiest fix was to require the target to not be in the
+        * sources, but there was an easier fix! I already empty the
+        * targets fot gcollapse, so I simply set that boolean to true
+        * (init_targ) when gegen was called with replace! This impacts
+        * the check in lines 489-492.
+
         cap confirm new variable `name'
         if ( _rc ) {
             local dummy `name'
             local rename ""
-            local addvar qui replace `dummy' = .
+            local addvar ""
+            local noobs qui replace `dummy' = .
             local retype = `retype' & 0
         }
         else {
             tempvar dummy
             local rename rename `dummy' `name'
             local addvar qui mata: st_addvar("`type'", "`dummy'")
+            local noobs  ""
             local retype = `retype' & 1
         }
     }
@@ -465,6 +486,13 @@ program define gegen, byable(onecall) rclass
                 global GTOOLS_CALLER "" di as err "{opth `ofcn'(varlist)} must call a numeric variable list."
                 exit _rc
             }
+
+            * See notes in lines 294-310
+            * if ( "`:list sources & dummy'" != "" ) { 
+            *     if ( "`replace'" != "" ) local extra " even with -replace-"
+            *     di as error "Variable `dummy' canot be a source and a target`extra'"
+            *     exit 198
+            * }
         }
     }
     else if ( `rc' == 0 ) {
@@ -578,6 +606,7 @@ program define gegen, byable(onecall) rclass
     }
     else if ( `rc' == 17001 ) {
         if ( "${GTOOLS_DUPS}" == "" ) {
+            `noobs'
             `rename'
             exit 0
         }

build/gisid.ado

@@ -3,7 +3,7 @@
 
 capture program drop gisid
 program gisid
-    version 13
+    version 13.1
 
     global GTOOLS_CALLER gisid
     syntax varlist            /// Variables to check

build/glevelsof.ado

@@ -3,7 +3,7 @@
 
 capture program drop glevelsof
 program glevelsof, rclass
-    version 13
+    version 13.1
 
     if ( `=_N < 1' ) {
         di as err "no observations"

build/gquantiles.ado

@@ -3,7 +3,7 @@
 
 capture program drop gquantiles
 program gquantiles, rclass
-    version 13
+    version 13.1
 
     if ( `=_N < 1' ) {
         error 2000