diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ac92564 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +*.so* +*.o +fn-fileget +fn-req +fn-req.static +pacparser.h +dist diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..d88d455 --- /dev/null +++ b/COPYING @@ -0,0 +1,9 @@ +All source files in this package called "frontier_client" that are +Copyright Fermilab are covered by the 2009 Fermitools license (a +BSD license), details in the file Fermilab-2009.txt + +There is a source file fn-base64.c that is covered by the python +license, and several in the "chashtable" directories covered by the +new BSD license. See the details in the files themselves. + +- Dave Dykstra, 25 June 2013 diff --git a/Fermilab-2009.txt b/Fermilab-2009.txt new file mode 100644 index 0000000..1b48b04 --- /dev/null +++ b/Fermilab-2009.txt @@ -0,0 +1,32 @@ +Fermilab Software Legal Information (BSD License) +Copied from http://fermitools.fnal.gov/about/terms.html + +Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of the FERMI NATIONAL ACCELERATOR LABORATORY, + nor the names of its contributors may be used to endorse or + promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/FrontierExceptionMapper.cpp b/FrontierExceptionMapper.cpp new file mode 100644 index 0000000..094b05d --- /dev/null +++ b/FrontierExceptionMapper.cpp @@ -0,0 +1,52 @@ +/* + * frontier client C++ exception mapper + * + * Author: Sinisa Veseli + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ +# +#include +#include +#include + +#include "frontier_client/FrontierException.hpp" +#include "frontier_client/FrontierExceptionMapper.hpp" + +namespace frontier { + +void FrontierExceptionMapper::throwException( + int errorCode, const std::string& errorMessage) { + switch(errorCode) { + case FrontierErrorCode_InvalidArgument: + throw InvalidArgument(errorMessage); + case FrontierErrorCode_MemoryAllocationFailed: + throw MemoryAllocationFailed(errorMessage); + case FrontierErrorCode_ConfigurationError: + throw ConfigurationError(errorMessage); + case FrontierErrorCode_SystemError: + throw SystemError(errorMessage); + case FrontierErrorCode_NetworkProblem: + throw NetworkProblem(errorMessage); + case FrontierErrorCode_ProtocolError: + throw ProtocolError(errorMessage); + case FrontierErrorCode_ServerError: + throw ServerError(errorMessage); + case FrontierErrorCode_UnknownError: + default: + throw FrontierException(errorMessage, errorCode); + } + throw FrontierException(errorMessage, errorCode); +} + +} // namespace frontier + + + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fb74278 --- /dev/null +++ b/Makefile @@ -0,0 +1,312 @@ +# +# frontier client Makefile +# +# Author: Sergey Kosyakov +# +# $Id$ +# +# Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY +# All rights reserved. +# +# For details of the Fermitools (BSD) license see Fermilab-2009.txt or +# http://fermitools.fnal.gov/about/terms.html +# + +FN_VER_MAJOR = 2 +FN_VER_MINOR = 9.1 +COMPILER_TAG = $(CC)_$(shell $(CC) -dumpversion) +PACKAGE_TAG = $(FN_VER_MAJOR).$(FN_VER_MINOR)__$(COMPILER_TAG) +SRC_PACKAGE_TAG = $(FN_VER_MAJOR).$(FN_VER_MINOR)__src + +ifndef EXPAT_DIR + EXPAT_LD_FLAGS= + EXPAT_INC = +else + EXPAT_LD_FLAGS= -L${EXPAT_DIR}/lib + EXPAT_INC = -I${EXPAT_DIR}/include +endif + +ifndef ZLIB_DIR + ZLIB_LD_FLAGS= + ZLIB_INC = +else + ZLIB_LD_FLAGS= -L${ZLIB_DIR}/lib + ZLIB_INC = -I${ZLIB_DIR}/include +endif + +ifndef OPENSSL_DIR + OPENSSL_LD_FLAGS= + OPENSSL_INC = +else + OPENSSL_LD_FLAGS= -L${OPENSSL_DIR}/lib + OPENSSL_INC = -I${OPENSSL_DIR}/include +endif + +# Don't need PACPARSER_LD_FLAGS because the library is loaded with dlopen +ifndef PACPARSER_DIR + PACPARSER_INC = +else + PACPARSER_INC = -I${PACPARSER_DIR}/include +endif + +# Mac OSX uses "dylib" instead of "so" for shared dynamic libraries +DYLIBTYPE := $(shell if [ -f /usr/lib/libc.dylib ]; then echo dylib; else echo so; fi) + +# GCC settings +DEBUG_OPTIM = -g -O2 +CFLAGS = $(DEBUG_OPTIM) +CXXFLAGS = $(DEBUG_OPTIM) +FNAPI_VERSION = $(FN_VER_MAJOR).$(FN_VER_MINOR) +CC = gcc +CXX = c++ +CXXOPT_LIB = -Wall $(CXXFLAGS) -DFRONTIER_DEBUG -DFNTR_USE_NAMESPACE -DFNTR_USE_EXCEPTIONS -fPIC -DPIC +CXXOPT_APP = -Wall $(CXXFLAGS) -DFRONTIER_DEBUG -DFNTR_USE_NAMESPACE -DFNTR_USE_EXCEPTIONS -fPIC -DPIC +COPT = -Wall $(CFLAGS) -DFRONTIER_DEBUG -fPIC -DPIC +LINK_SO = c++ $(CXXFLAGS) -shared -o libfrontier_client.so.$(FN_VER_MAJOR).$(FN_VER_MINOR) -Wl,-soname,libfrontier_client.so.$(FN_VER_MAJOR) +LINK_DYLIB = c++ $(CXXFLAGS) -dynamiclib -undefined dynamic_lookup -compatibility_version $(FN_VER_MAJOR) -current_version $(FN_VER_MAJOR).$(FN_VER_MINOR) -o libfrontier_client.$(FN_VER_MAJOR).$(FN_VER_MINOR).dylib -single_module $(LIBS) +LINK = ar cr libfrontier_client.a + +COMMON_INC = ${EXPAT_INC} ${OPENSSL_INC} ${PACPARSER_INC} ${ZLIB_INC} +INC = ${COMMON_INC} -I./include -I. +LIBDIR = +# -ldl needed by Fedora 18 +LIBS = ${EXPAT_LD_FLAGS} -lexpat ${OPENSSL_LD_FLAGS} -lssl -lcrypto -L. ${ZLIB_LD_FLAGS} -lz -ldl -lrt + +OBJ = fn-base64.o fn-hash.o fn-zlib.o frontier.o response.o payload.o rs-bin.o statistics.o memdata.o \ + frontier_log.o frontier_error.o frontier_config.o fn-mem.o pacparser-dlopen.o anydata.o frontier-cpp.o FrontierExceptionMapper.o +HDR = include/frontier_client/frontier.h fn-internal.h http/fn-htclient.h include/frontier_client/frontier_error.h include/frontier_client/frontier_log.h +HDRXX = include/frontier_client/frontier-cpp.h include/frontier_client/FrontierException.hpp include/frontier_client/FrontierExceptionMapper.hpp +BINAPP = fn-req fn-fileget +BINSCRIPTS = fnget.py frontierqueries + +ifeq ($(DYLIBTYPE),dylib) + LIBCLIENT = libfrontier_client.dylib + STATICBINAPP = +else + LIBCLIENT = libfrontier_client.so + STATICBINAPP = fn-req.static +endif + + + +all: htclient $(LIBCLIENT) $(BINAPP) $(STATICBINAPP) + + + +##################################################### +### BEGIN KIT RELATED STUFF. Note: requires gmake :-( + +package := frontier_client +tmpdir := $(shell pwd)/tmp +installdir := $(shell pwd)/dist +distdir := $(installdir) +files := ./RELEASE_NOTES ./bin/* ./lib/*.$(DYLIBTYPE)* ./include/frontier_client/*.* +src_files = `sh -c "find . \\( \\( -name CVS -o -name SCCS -o -name tmp \\) -prune \\) -o \\( -type d -links 2 -print -prune \\) -o \\! -type d ! -name '.manifest*' ! \\( -name core -type f \\) -print 2>/dev/null | sort"` +distfile := $(tmpdir)/$(package)__$(PACKAGE_TAG).tar +distfile_gz := $(distfile).gz +src_distdir := $(package)__$(SRC_PACKAGE_TAG) +src_distfile := $(tmpdir)/$(src_distdir).tar +src_distfile_gz := $(src_distfile).gz + +# Build all distributions from scratch: +all-dist: clean src-dist dist + +# Source distribution. +src-dist: clean $(tmpdir) + (cd http && $(MAKE) clean) + tar cvf $(src_distfile) $(src_files) + mkdir -p $(tmpdir)/$(src_distdir) + (cd $(tmpdir)/$(src_distdir) && tar xvf $(src_distfile)) + rm -f $(src_distfile) + (cd $(tmpdir) && tar zcvf $(src_distfile_gz) $(src_distdir)) + rm -rf $(tmpdir)/$(src_distdir) + + +dist-clean: + rm -rf $(tmpdir) $(distdir) + +$(tmpdir) $(distdir): + mkdir -p $@ + +dist: $(distfile_gz) + +$(distfile_gz): $(tmpdir) install + @echo "Making $@" + rm -f $(distfile); (cd $(distdir); tar cvf $(distfile) $(files)) + rm -f $(distfile_gz); gzip $(distfile) + +install: $(distdir) all + mkdir -p $(distdir)/lib + mkdir -p $(distdir)/bin + rm -f $(distdir)/lib/lib*.$(DYLIBTYPE)* +ifeq ($(DYLIBTYPE),dylib) + cp libfrontier_client.$(FN_VER_MAJOR).$(FN_VER_MINOR).dylib $(distdir)/lib + (cd $(distdir)/lib/ && ln -s libfrontier_client.$(FN_VER_MAJOR).$(FN_VER_MINOR).dylib libfrontier_client.$(FN_VER_MAJOR).dylib ) + (cd $(distdir)/lib/ && ln -s libfrontier_client.$(FN_VER_MAJOR).$(FN_VER_MINOR).dylib libfrontier_client.dylib ) +else + cp libfrontier_client.so.$(FN_VER_MAJOR).$(FN_VER_MINOR) $(distdir)/lib + (cd $(distdir)/lib/ && ln -s libfrontier_client.so.$(FN_VER_MAJOR).$(FN_VER_MINOR) libfrontier_client.so.$(FN_VER_MAJOR) ) + (cd $(distdir)/lib/ && ln -s libfrontier_client.so.$(FN_VER_MAJOR).$(FN_VER_MINOR) libfrontier_client.so ) +endif + cp -r include $(distdir) + cp $(BINAPP) $(distdir)/bin + chmod 755 $(BINSCRIPTS) + cp $(BINSCRIPTS) $(distdir)/bin + cp RELEASE_NOTES $(distdir) + + +### END KIT RELATED STUFF. +##################################################### + +htclient: + (cd http && $(MAKE) CC="$(CC)" COPT="$(COPT)" COMMON_INC="$(COMMON_INC)" all) + +libfrontier_client.so: $(OBJ) http/.libs + rm -f libfrontier_client.so + rm -f libfrontier_client.so.$(FN_VER_MAJOR) + rm -f libfrontier_client.so.$(FN_VER_MAJOR).$(FN_VER_MINOR) + rm -rf .libs + mkdir .libs + cp ./http/.libs/*.o .libs + cp $(OBJ) .libs + $(LINK_SO) .libs/*.o $(LIBS) + ln -s libfrontier_client.so.$(FN_VER_MAJOR).$(FN_VER_MINOR) libfrontier_client.so.$(FN_VER_MAJOR) + ln -s libfrontier_client.so.$(FN_VER_MAJOR).$(FN_VER_MINOR) libfrontier_client.so + +libfrontier_client.dylib: $(OBJ) http/.libs + rm -f libfrontier_client.dylib + rm -f libfrontier_client.$(FN_VER_MAJOR).dylib + rm -f libfrontier_client.$(FN_VER_MAJOR).$(FN_VER_MINOR).dylib + rm -rf .libs + mkdir .libs + cp ./http/.libs/*.o .libs + cp $(OBJ) .libs + $(LINK_DYLIB) .libs/*.o + ln -s libfrontier_client.$(FN_VER_MAJOR).$(FN_VER_MINOR).dylib libfrontier_client.$(FN_VER_MAJOR).dylib + ln -s libfrontier_client.$(FN_VER_MAJOR).$(FN_VER_MINOR).dylib libfrontier_client.dylib + + +fn-base64.o: fn-base64.c $(HDR) fn-base64.h + $(CC) $(COPT) $(INC) -c fn-base64.c + +fn-hash.o: fn-hash.c $(HDR) fn-hash.h chashtable/hashtable.c chashtable/hashtable.h + $(CC) $(COPT) $(INC) -c fn-hash.c + +fn-zlib.o: fn-zlib.c $(HDR) fn-zlib.h fn-base64.h + $(CC) $(COPT) $(INC) -c fn-zlib.c + +fn-mem.o: fn-mem.c $(HDR) + $(CC) $(COPT) $(INC) -c fn-mem.c + +frontier.o: frontier.c $(HDR) fn-zlib.h + $(CC) -DFNAPI_VERSION="\"$(FNAPI_VERSION)\"" $(COPT) $(INC) -c frontier.c + +pacparser-dlopen.o: pacparser-dlopen.c $(HDR) + $(CC) $(COPT) $(INC) -c pacparser-dlopen.c + +payload.o: payload.c $(HDR) + $(CC) $(COPT) $(INC) -c payload.c + +response.o: response.c $(HDR) + $(CC) $(COPT) $(INC) -c response.c + +rs-bin.o: rs-bin.c $(HDR) + $(CC) $(COPT) $(INC) -c rs-bin.c + +statistics.o: statistics.c $(HDR) + $(CC) $(COPT) $(INC) -c statistics.c + +memdata.o: memdata.c $(HDR) fn-zlib.h + $(CC) $(COPT) $(INC) -c memdata.c + +frontier_error.o: frontier_error.c include/frontier_client/frontier_error.h include/frontier_client/frontier_log.h + $(CC) $(COPT) $(INC) -c frontier_error.c + +frontier_log.o: frontier_log.c include/frontier_client/frontier_log.h + $(CC) $(COPT) $(INC) -c frontier_log.c + +frontier_config.o: frontier_config.c $(HDR) + $(CC) -DFNAPI_VERSION="\"$(FNAPI_VERSION)\"" $(COPT) $(INC) -c frontier_config.c + +main.o: main.c $(HDR) + $(CC) $(COPT) $(INC) -c main.c + +main: all main.o + $(CXX) -g -Ldist/lib -o main main.o -L. -lfrontier_client + +test-b64url.o: test-b64url.c $(HDR) + $(CC) $(COPT) $(INC) -c test-b64url.c + +test-b64url: test-b64url.o + $(CXX) -g -L. -o test-b64url test-b64url.o -L. -lfrontier_client + +anydata.o: anydata.cc $(HDRXX) $(HDR) + $(CXX) $(CXXOPT_LIB) $(INC) -c anydata.cc + +frontier-cpp.o: frontier-cpp.cc $(HDRXX) $(HDR) + $(CXX) $(CXXOPT_LIB) $(INC) -c frontier-cpp.cc + +maincc.o: maincc.cc $(HDRXX) + $(CXX) $(CXXOPT_APP) -I./include -c maincc.cc + +fn-maincc: maincc.o $(HDRXX) + $(CXX) $(CXXOPT_APP) -L. -o fn-maincc maincc.o -L. -lfrontier_client + +maintest.o: maintest.cc $(HDRXX) + $(CXX) $(CXXOPT_APP) -I./include -c maintest.cc + +fn-maintest: maintest.o $(HDRXX) + $(CXX) $(CXXOPT_APP) -L. -o fn-maintest maintest.o -L. -lfrontier_client + +FrontierExceptionMapper.o: FrontierExceptionMapper.cpp $(HDRXX) + $(CXX) $(CXXOPT_APP) -I./include -c FrontierExceptionMapper.cpp + +getcids.o: getcids.cc $(HDRXX) + $(CXX) $(CXXOPT_APP) -Idist/include -c getcids.cc + +getcids: all getcids.o $(HDRXX) + $(CXX) $(CXXOPT_APP) -Ldist/lib -o getcids getcids.o -L. -lfrontier_client + +request_each.o: request_each.cc $(HDRXX) + $(CXX) $(CXXOPT_APP) -Idist/include -c request_each.cc + +request_each: all request_each.o $(HDRXX) + $(CXX) $(CXXOPT_APP) -Ldist/lib -o request_each request_each.o -L. -lfrontier_client + +test-pescalib.o: test-pescalib.cc $(HDRXX) + $(CXX) $(CXXOPT_APP) -I./include -c test-pescalib.cc + +fn-pescalib: test-pescalib.o $(HDRXX) + $(CXX) $(CXXOPT_APP) -L. -o fn-pescalib test-pescalib.o -L. -lfrontier_client + +test-any.o: test-any.cc $(HDRXX) + $(CXX) $(CXXOPT_APP) -I./include -c test-any.cc + +fn-any: test-any.o $(HDRXX) + $(CXX) $(CXXOPT_APP) -L. -o fn-any test-any.o -L. -lfrontier_client + +test-req.o: test-req.cc $(HDRXX) + $(CXX) $(CXXOPT_APP) -I./include -c test-req.cc + +fn-req: test-req.o $(HDRXX) + $(CXX) $(CXXOPT_APP) -L. -o fn-req test-req.o -L. -lfrontier_client + +# staticly linked app works better in gdb +fn-req.static: libfrontier_client.so fn-req + $(CXX) $(CXXOPT_APP) -L. -o fn-req.static test-req.o .libs/*.o $(LIBS) + +fn-fileget.o: fn-fileget.c $(HDR) + $(CC) $(COPT) $(INC) -c fn-fileget.c + +fn-fileget: fn-fileget.o + $(CC) $(COPT) $(INC) -o fn-fileget fn-fileget.o -L. -lfrontier_client + + +clean: + rm -f *.o *.core core libfrontier_client.* frontier main getcids request_each fn-any $(BINAPP) $(STATICBINAPP) + (cd http && $(MAKE) clean) + rm -rf .libs + rm -rf $(tmpdir) + rm -rf $(distdir) + + diff --git a/README b/README new file mode 100644 index 0000000..4ae628f --- /dev/null +++ b/README @@ -0,0 +1,23 @@ +$Id$ + +1. To compile: +make + +2. To use in C application: +#include + +Link against -lfrontier_client + +3. To use in C++ application: +#include + +Link against -lfrontier_client + +4. Environment: +export FRONTIER_SERVER[1234]=frontier_server +export FRONTIER_PROXY[1234]=local_proxy +export FRONTIER_LOG_LEVEL={nolog,info,debug} + + + +For more details, guides/manuals see http://frontier.cern.ch diff --git a/RELEASE_NOTES b/RELEASE_NOTES new file mode 100644 index 0000000..28ad762 --- /dev/null +++ b/RELEASE_NOTES @@ -0,0 +1,873 @@ +v2_9_1 - 19 Mar 2020 (dwd) + - Remove extra slash at the beginning of all http queries, introduced + in 2.8.21 (with the ipv6 square brackets feature). + - Make "Name or service not known" warning messages for proxyconfigurls + into debug messages, since it is recommended to try grid-wpad and wpad + even though in most cases the names do not exist. + +v2_9_0 - 22 Jan 2020 (dwd) + - Add the frontier_statistics C API. Collects the number of queries, + the number of errors in fetches, and the min, max, and average of + both the number of bytes and elapsed milliseconds of queries. + When debug log level is enabled, the statistics are printed at the + end of the log. Statistics are only collected when debugging is + enabled or after frontier_statistics_start() is called. + - Update fn-req to use the statistics API by reporting the kilobytes + per second of the transfer. + +v2_8_21 - 20 Dec 2019 (dwd) + - Fix a crash during print of the fatal error message when someone + specifies a proxyconfigurl without a serverurl. + - Use "Cache-Control: no-cache" instead of "Pragma: no-cache" when a + cache entry needs a hard refresh. This change is needed for the + default configuration of squid-3 (and later) which ignores Pragma + when there is a Cache-Control header (and frontier-client always + sends one). + - Randomize the starting addresses used from round-robin DNS lookups. + - Fix compilation on centos8. + - Allow "http://" to be missing from URLs. This change is to support + the proxy auto config standard which allows http:// to be missing + in the return PROXY. + - Allow square brackets to surround the host name in URLs, in order + to be able to parse IPv6 IP addresses without confusing the colons + in them with the beginning of a port number. + - In debug, warning, and error messages that include IP addresses, + put IPv6 addresses in square brackets and add colon and port + number when it is known. + - Add lower level details in the warning message about retrying after + a system error. + - Extend fn-fileget to read multiple chunks of data instead of one + big chunk. + - Add elapsed seconds to the frontierqueries output and the -secsfirst + option to order it first. + +v2_8_20 - 3 Nov 2016 (dwd) + - Fix bug introduced in v2_8_15 that caused a segmentation fault + when a DNS name lookup of proxies fail. The fault happened in + the function frontierHttpClnt_usinglastproxyingroup(). + - When both proxyconfigurl and proxyurl options are used, change the + priority so that proxies returned from proxyconfigurl are used + before proxyurls. It used to be the opposite order. + - When proxyconfigurl options are used, ignore loadbalance=proxies. + +v2_8_19 - 12 Apr 2016 (dwd) + - Fix bug in the function that gets the name & IP address of servers + and proxies as a string. This bug was introduced in v2_8_15. The + function is supposed to return the host name followed by the IP + address in square brackets, however for proxies and for direct- + connect servers it was returning only the IP address in square + brackets. This mostly affects warning & debug messages, but also + causes the digital signature checking option (security=sig) to + fail on direct-connect servers. + - Add support for $X509_CERT_DIR. If it is set and the capath option + is not set, it will be used as the directory containing CA certs + and CRLs instead of the default /etc/grid-security/certificates. + +v2_8_18 - 16 Feb 2016 (dwd) + - Change the ordering of the linking options on the frontier client + shared library, to get it to build properly on Ubuntu 12.04. + +v2_8_17 - 15 Feb 2016 (dwd) + - Move SSL cleanup function calls from when a frontier channel is + deleted to the library finalizer, which is called by gcc when a + shared library is unloaded. The cleanup calls in channel delete + were interfering with other uses of SSL in ATLAS. + - Fix bug/compiler warning introduced in 2.8.15 that prevented + proxyconfigurl=file://... from working. + +v2_8_16 - 4 Feb 2016 (dwd) + - Initialize the client loadbalance feature with a random number from + OpenSSL instead of getpid(), because when using cgroups all of the + process ids can be the same. + +v2_8_15 - 2 Feb 2016 (dwd) + - Fix bug present since 2.8.6 that considered a whole proxy group + failed if the last one failed, even if not all of the proxies in + the group had failures. That could happen after a round-robin + group had been "shuffled" for load balancing. + - Slightly change the retry strategy to treat connection refused and + network unreachable errors the same as connection timeouts. + - Refactor the http client code to do a lot more sharing of code + between servers and proxies. + - Treat IPv4 and IPv6 addresses present in the same DNS name as two + separate groups; in other words, when load balancing between + different addresses in a round-robin DNS name, stay within one IP + family. Start with the preferred IP family and only advance to + the other IP family when all the addresses in that family had + errors. If preferipfamily=0, the preferred family is considered + to be the family of the first address returned by the operating + system. + - Fix bug present since 2.8.9 where if a query did not send an Age + header because it was a fresh query, but a previous query on the + same connection did have an old Age header, a max-age exceeded + condition could be erroneously triggered and cause undesired + soft and hard refreshes. + - Treat an http error code 404 as a server error just like 5XX errors, + rather than a protocol error. That means the proxy will be assumed + to be good and the server will be advanced. + - Export the common includes in the Makefile to http/Makefile, + because http source files include fn-internal.h which includes + openssl files which may not be in the standard places. Requested + by CMS. + +v2_8_14 - 14 Oct 2015 (dwd) + - Fix bug introduced in v2_8_13 of using memory after it was freed. + The problem only showed up after running for 5 minutes, after + address info was freed in order to look up names in the DNS again. + +v2_8_13 - 9 Oct 2015 (dwd) + - Support IPv6 addresses. + - Add new complex connection string option "preferipfamily" which + controls the ordering of IP addresses tried when both IPv4 and + IPv6 values are present in a DNS lookup. When set to "6", prefers + IPv6. When set to "4", prefers IPv4. When set to "0", uses the + order set by the operating system. Default is "4". When attempts + fail, all values are tried as before; this only controls the order. + +v2_8_12 - 13 Jul 2015 (dwd) + - Replace unhelpful "inconsistent result from poll" message that can + happen in 2.8.11 on a connect failure with more helpful "Connection + refused" or "Network unreachable". + +v2_8_11 - 27 Aug 2014 (dwd) + - Convert from using select() to poll() in order to be able to work + when more than FD_SETSIZE (1024) file descriptors are open. + - Move "extern"s for global variables to fn-internal.h instead of + separately declared in source files. + +v2_8_10 - 30 Jan 2014 (dwd) + - Correct segmentation fault caused by an incomplete conversion to + using ctime_r() instead of ctime(). This happens any time a job + requests data more than 5 minutes after the first time it requests + data. + +v2_8_9 - 2 Jan 2014 (dwd) + - Add support for a "forever" time-to-live ("ttl") for cache + expirations in addition to the previous "short" ttl. The server + determines the amount of time to assign to each ttl level, but the + client determines which one to apply to each query. Add new C++ + API function Connection::setTimeToLive() and C API function + frontier_setTimeToLive() which take a "ttl" parameter that is 1 + for short, 2 for long, and 3 for forever. The default ttl is 2. + Deprecate the Connection::setReload() and frontier_setReload() API + functions where a true value is equivalent to setting a ttl to 1 + with one of the new API functions. + - Extend the forcereload complex connection string option and + FRONTIER_FORCERELOAD environment variable so that a "long" or + "softlong" applies to short & long ttl only, and a "forever" or + "shortforever" applies to all ttls. (By contrast the freshkey + option/FRONTIER_FRESHKEY variable is not extended to support + forever ttl). + - If the server requests a maximum cache age in the response payload + (not to be confused with the normal max cache age in the http + header), or if there was a protocol error (with a default maximum + cache age of 5 minutes), and the response is older than the + maximum age according to the http "Age" header from squid, then + first try a soft retry with that maximum cache age. Formerly it + used to always do a soft retry with maximum cache age of 0 seconds + immediately after a protocol error. This change avoids making a + server error condition worse by causing a flash overload with many + clients requesting retries at the same time. If the soft retry + still returns a protocol error, continue to do a hard retry like + it used to. New frontier servers (servlet version 3.33 or later) + also request a maximum cache age (default 5 minutes) on all empty + query responses because those can indicate temporary problems even + if they're not technically errors. + - Change fnget.py so it doesn't always request &sec=sig, since that + fails with newer servers that have no host certificates. Instead, + add --sign option which appends the &sec=sig. + - Add '-R' option to fn-req to request "forever" time-to-live. Add + the same option to fn-fileget and also the '-r' option that was + on fn-req for a "short" time-to-live. + - Change fn-fileget to honor the FRONTIER_RETRIEVEZIPLEVEL environment + variable and the retrieve-ziplevel complex connection string option, + except that unlike the regular frontier client the default is 0. + - Change fn-fileget to doubly URL-encode any characters in the file + path that aren't unreserved (that is, anything other than + alphanumeric, dash, underscore, forward slash, and period). URL + encoding replaces the other characters with a % and 2 hex digits + of their ascii representation. The % is also encoded (as %25) to + make it doubly-encoded, because it is decoded twice by the time + it reaches the Frontier servlet file plugin. + - When calculating the secure hash on the requested URL, do one + level of URL decoding (turning % and 2 hex digits into its ascii + equivalent) before calculating the hash, in order to match the + server which calculates the URL hash after one decoding. + - Extend the frontierqueries tool to put a '+' in the first column + for "forever" ttl queries similar to the '*' that's put there for + "short" ttl queries. + - Use ctime_r() instead of ctime() to get debug time stamps because + the latter is not thread-safe. + +v2_8_8 - 9 Jul 2013 (dwd) + - Fixed the exit code from the fn-fileget command so it returns a + zero on success and a one on failure rather than the opposite. + +v2_8_7 - 28 Jun 2013 (dwd) + - Added support for digital signatures on responses. This is enabled + by the connection string option "(security=sig)". This option + requires a directory of Certifying Authorities certificates and + CRLs which can be set with "(capath=/path/to/dir)". The default + path is /etc/grid-security/certificates. Also requires frontier + servlet version of at least 3.30. The signature is made up of a + SHA256 checksum of the requested URL plus the response, encrypted + with the RSA private key of the server's X509 host key. The + server's X509 certificate is read from the server and verified by + openssl. Either the certificate's primary subject name or one of + its alternate DNS names is required to match the host name in the + serverurl. The signature is decrypted with the certificate's RSA + public key, and the resulting value is required to match the + calculated SHA256 checksum of the URL plus the response. + - Fix a segfault that happened when the application tries to do + another query after an error with all proxies & servers. The + problem was introduced in the last release. + - Removed internal MD5 implementation and switched to use openssl's + implementation instead. + - Added -ldl to the library link line because that's needed by + Fedora 18. + +v2_8_6 - 29 Apr 2013 (dwd) + - Change retry strategy so that if a proxy or directly-connected server + translates to a round-robin DNS name and there are errors, all of the + IP addresses will be tried before giving up. NOTE: this does not + extend to the case of servers being contacted through proxies, so + it is best to list all of the individual servers in a round-robin + service separately (after the round-robin name) in order to ensure + that all are tried through proxies. + - Proxy connect timeouts are now given special treatment, immediately + identifying them as proxy errors. + - Change retry strategy so that non-server errors will first try + every server with every proxy in a proxy group (those that have not + had connect timeouts), then every server with every proxy in the + next group, etc. A proxy group is defined as either all proxyurls + if loadbalance=proxies or all IP addresses in a round-robin name in + a proxyurl or backupproxyurl. Previously every proxy was tried + with the first server regardless of proxy group, then the next + server, etc, which was not optimal for far-away backup proxies. + - Whenever a request is made more than 5 minutes since the first one, + any current connection is now dropped, the proxy list is reset to + the beginning, all cached DNS names of proxies are cleared, all + proxy error flags are cleared, and the start time is reset so the + process can be done again 5 minutes later. Servers are treated + similarly every 30 minutes. Formerly, only the DNS name caches and + the corresponding error flags were cleared whenever 5 minutes had + elapsed. + - The full current retry strategy is now documented in detail at + https://twiki.cern.ch/twiki/bin/view/Frontier/ClientRetryStrategy + - Add new connection string parameter "proxyconfigurl" which is a URL + to an internet-standard Proxy Auto Config (PAC) file for locating a + list of proxies. Parameter may appear more than once, so that if a + config server can't be reached the next one is tried. Each URL may + indicate a round-robin name, and if so all of the servers are tried. + The special value "auto" translates to "http://wpad/wpad.dat" which + is commonly used for the Web Proxy Auto Discovery standard. Requires + pacparser library with a version greater than 1.3.0. Since the + library is over a megabyte of code, it is loaded with dlopen. The + url for this may be a "file://" type but if that fails it is a + fatal error. Also the client IP address for matching inside the + PAC file is determined in a less reliable manner with file:// URLs. + Can specify a directory for the pacparser library install with + the PACPARSER_DIR make variable. + - Add new environment variable FRONTIER_PROXYCONFIGS which is a + semicolon-separated list of proxyconfigurl values. + - Add new connection string parameter "maxagesecs=N" which sets the + header "Cache-control: max-age=N" so the client can control the + maximum cache age seconds when there's no reloading happening. + Setting "maxagesecs=0" is equivalent to "forcereload=softlong". + - If a memory allocation failure happens while retrieving data, return + immediately instead of trying all proxies and servers. + - Insert "mem_alloc: failed" messages in several places that handled + malloc failures but didn't set an error message. + - In the place most likely to get a memory allocation failure + (allocating for an entire uncompressed response), change the + message to also report the number of bytes that were attempted to + be allocated. + - Free a small malloced space that was leftover when reading the + subject name from $X509_USER_PROXY. + +v2_8_5 - 13 Dec 2011 (dwd) + - Fix bug that caused a segmentation fault when allocating the large + piece of memory for the query response failed. + - Fix bug where only the first channel accessed after a process forked + would close the socket connection, causing responses to get mixed up + between different processes. Now it will close & reopen all channels + accessed. + +v2_8_4 - 21 Jul 2011 (dwd) + - Fix bug that caused outgoing http connections to remain open after + the frontier client object was deleted, unless the last query + happened to have been a large one (>16kbyte). This bug has existed + for 4-1/2 years, ever since persistent connections were implemented. + Most production code only uses one frontier client object so that's + probably why it wasn't noticed. The problem was noticed in a CMS + tool that created and deleted the frontier object many times. + +v2_8_3 - 12 Jul 2011 (dwd) + - Fix bug that caused 'unzip unknown error' under relatively rare + conditions. The error code was actually reporting a normal + condition, it just wasn't being recognized as such. + +v2_8_2 - 30 Jun 2011 (dwd) + - Update the retry strategy so that when an error is not clearly a + server error (server errors imply that the proxy was good), try every + proxy with every server in turn. Previously, non-server errors + would cause the strategy to do direct connections to all servers + after every proxy failed with just the first server. The old + strategy was fine for CMS Offline where the first server is a + round-robin between all servers, but not good for CMS Online or + ATLAS where that is not the case. + - Make protocol error-induced reloads try "Cache-control: max-age=0" + before "Pragma: no-cache", because that's gentler on the servers + since it only asks for the modification times to be immedately + checked (that is, it revalidates the cache). $FRONTIER_FORCERELOAD + variable still by default uses "Pragma: no-cache", but now if the + value begins with "soft" (that is, "softlong" or "softshort") it + uses the more gentle refresh. + - Do md5 calculations and unzipping (when needed) as the data is + received rather than waiting until all data is received . This + gives a slight reduction in elapsed time because those calculations + can be interleaved with I/O. + - Turn low-level error messages into just debug messages when they are + caught at a higher level and retried. At the retrying level they + were already being printed as warning messages but the output was + confusing because it showed both 'error' and 'warn'. + - Include ServerError in the types of C++ exceptions that may be thrown. + Previously it would have been mapped to an UnknownError exception. + +v2_8_1 - 24 Feb 2011 (dwd) + - Add small fn-fileget command line tool that can retrieve files from + frontier servlets (release 3.28 or later) that are configured to + serve files instead of database queries. + Usage: fn-fileget [-c connect_string] filepath ... + Honors all regular frontier environment variables, although it + doesn't support retrieve-ziplevel because it's assumed that files + can be pre-compressed. + +v2_8_0 - 17 Dec 2010 (dwd) + - The second digit in this release changed because the FrontierConfig + structure changed. This only affects binary compatibilty for the + C interface, not the C++ interface used by all current users of + the frontier client. + - Fix segmentation fault that happens when there are no proxies + defined and the server gets errors. Seen in ATLAS. + - Fix segmentation fault that happens when all proxies have had + errors but then a direct connect to server succeeds followed by + an error on the same persistent connection. Seen in ATLAS. + - Change to using a '/' as a URL delimiter instead of '?' because the + the latter is not officially compatible with caching according to + the HTTP RFCs and requires a non-default squid configuration. + Requires Frontier servlet version 3.24 or greater. + - Put contents of new environment variable $FRONTIER_ID into the + beginning of the X-Frontier-Id header which is logged in squid and + the frontier servlet. Overrides including $CMSSW_VERSION if it was + set. + - Add new complex connection string option "backupproxyurl" which is + very similar to proxyurl except that the specified proxies are + always tried after all proxyurls and they're not included in the + client load balancing done when loadbalance=proxies is set. This + is intended to be used to identify proxies physically located near + the servers whose purpose is to catch client failures without + interfering with server operation. Implies failovertoserver=no. + - Increase maximum number of allowed proxies from 16 to 24. The + maximum number of allowed servers is still 16. + - Change the re-try strategy so that when the last server experiences + a server error but not all proxies have yet had errors, continue + trying with the next proxy. It used to fail even if more proxies + were available. + - Add printing of IP addresses (when known) in addition to names in + the warning messages that identify proxies and servers. This helps + in identifying the selected addresses in round-robins when there + are problems. + - Add -lcrypto to the libs because it is needed when being built + by gcc45. Previously it was pulled in as a dependency of -lssl. + Requested by Peter Elmer. + +v2_7_15 - 14 May 2010 (dwd) + - Set clientcachemaxresultsize=0 by default, disabling the client + cache. + +v2_7_14 - 28 April 2010 (dwd) + - Support compilation on Mac OSX 10.6 Snow Leopard by changing link + options "-flat-namespace" to "-undefined dynamic_lookup". In + order to make that to then work on 10.5 Leopard, removed + "-undefined suppress". + - Link libexpat into libfrontier_client shared library so it + becomes an implicit dependency. + - Remove the default building of a local copy of the expat library + and instead assume the library is available in the operating system + if the EXPAT_DIR command line isn't provided. + - Clean up linking of applications in the Makefile to no longer + explicitly include libraries which are implicitly pulled in by + the libfrontier_client shared library. + +v2_7_13 - 5 February 2010 (dwd) + - Support forking of the parent process by checking for a change in + process ID before every query and if it has then drop the socket + connection, close the debug log, and update the X-Frontier-ID header. + - Added support for an optional '%P' in the debug log file name which + is replaced by the current process id, including after a fork. + - Added '-F N' option to fn-req tool to fork after N iterations (which + are specified by '-C N'), in order to test forking. + - Remove trailing newlines from the 'encoding request' debug message + (they were already removed from the query). + +v2_7_12 - 13 January 2010 (dwd) + - Add a new freshkey option to complex connection strings. Anything + in the value is added to the URLs used for queries, so that if a new + value is used the queries will be guaranteed to not be in a cache. + If the value begins with "long" that is removed and the rest of the + value is added to all queries, otherwise the value is added to only + "short" time-to-live queries. If not present in the connection + string, may come from environment variable FRONTIER_FRESHKEY. + Default is no extra key. + - Avoid a possible segmentation fault when a connection socket needs + to be re-opened but the open fails. + - Avoid a segmentation fault when getpwent() fails (due to a problem + in the operating system). + - Include current proxy and server name in some http-level error + messages where it might be helpful to know. + - Separate out install target from dist target, and make the dist target + work even when 'distdir' is not the default. The directory can now + be set with either installdir= or distdir= on the make command line. + - Change fn-req tool to handle more than one command line option at + once. + - Change fnget.py tool to handle keepalive messages that can come from + the frontier server for queries that take a long time to service. + - Switch from the GNU Lesser General Public License to the Fermitools + (BSD) license. + +v2_7_11 - 18 June 2009 (dwd) + - Whenever a system error happens on a proxy or server connection, + retry query once on a new connection. This works around a + 'connection reset by peer' problem seen on a percentage of CMS HLT + 8-core nodes when 7 parallel applications connecting to localhost + squids was tried for the first time. + - Add new failovertoserver option to complex connection strings. + When set equal to no, direct connections to servers will not be + attempted after failures contacting a proxy. Default yes. + - Remove the little fn-any application from the installation bin. + +v2_7_10 - 01 June 2009 (dwd) + - Use inflate zlib API instead of uncompress when uncompressing + payloads, analogous to the deflate change put in v2_7_9. Pete + Elmer requested this too although he said it wasn't as important + as deflate. + - If 5 minutes has elapsed since a DNS name was looked up and the + name is needed again, re-lookup the name. This also erases + the information about whether a particular address had an error so + failing addresses in a round-robin DNS will be tried once again. + - Added the destination IP address to read and write timeout warning + messages. + - Changed debug log output to close the log file only when the + frontier channel is closed, not after every message. This makes + logging to AFS more than an order of magnitude faster, making the + performance impact of logging to AFS now imperceptible. + - Added printing the frontier client version number at the beginning of + debug logs. + - Added frontierqueries and fnget.py scripts to installation bin. + - Added -c option to fn-req test application to repeat the query a + specified number of times on the same connection. This is useful + for load testing with small queries (as long as they are larger + than clientcachemaxresultsize). + +v2_7_9 - 10 October 2008 (dwd) + - Use deflate zlib API instead of compress2 when compressing URLs in + order to keep it initialized and avoid many large (~250KB) + allocations spread out over a run. Requested by Pete Elmer. + - Treat 5XX HTTP errors as server errors instead proxy errors, + so the retry algorithm will move on to the next server instead + of the next proxy + - Send 'Cache-Control: max-stale=1' header on every request, as an + indication to the frontier server that the 5XX errors are handled + properly and it may send back 'Cache-Control: stale-if-error=1' + which tells squid 2.7 to return a 504 error if the server later + goes down after the data is expired rather than returning stale + data. + - Fixed a segmentation fault that can happen if an application tries + to do another query after an error that used up all servers + occurred (COOL sometimes does this). + - Added some extra includes in C++ files for gcc 4.3 + +v2_7_8 - 20 June 2008 (dwd) + - Changed the retry strategy again, because the one introduced in + v2_7_5 last December did not correctly handle certain error + conditions. This becomes especially apparent with the + (in-progress) upgrade to the frontier server (the upgrade that + improves cache consistency and greatly reduces the time for + database changes to reach clients); some types of errors would + have never been cleared from the caches with that server upgrade. + As part of this new strategy: + - Separated out the kind of protocol error that is explicitly + reported by a server (which is used almost exclusively for + database errors) to be a new type called server error. + - Repaired an inadvertent (but as it turns out, useful for + backward-compatibility in the new server upgrade) reporting of + other types of errors on the server (known in the client-server + protocol as a global error) from being an undefined error to + being a protocol error. + The new strategy is now this: + - If it was a server error, use only one proxy and cycle through + the servers. Don't do any cache reloads because the error will + expire from the cache in 5 minutes (even with the current + frontier server) and we want it to stay there to avoid + overloading the server. This is most like the former handling + for all protocol errors, except that it used to also have a + last-resort cache reload for all servers. + - Else if it was a protocol error, cycle through the proxies + using the first server and then cycle through direct connects + to servers, attempting a reload after every try. This is + exactly like the original, pre-v2_7_5 strategy that was used + for all errors and will flush the cache of errors that + have been repaired. This type of error is much less common + than server errors. + - Else do the same as protocol error except with no cache reloads. + This covers the cases of networking problems, machines that are + down, and proxy or server overloads without making overload + situations much worse by doing many reloads. + - Added tracking of errors on individual addresses listed in round- + robin DNS, just like did before when doing client-controlled load + balancing of proxies or servers, so that those addresses are not + reused except when all proxies have had at least one failure. + - When there's a hit in the local client cache, stopped "shuffling" + proxies and servers (that is, advancing the round-robin or + selecting a new random address when load balancing). That was + only intended to be done just before opening a new connection to a + proxy or server. + - Fixed problem with client-controlled load balancing that caused a + failure on one address to give preferential treatment to the + working address listed after it; for example, if there were 3 + servers and one had a failure, one of the remaining servers got + 2/3 of the load and the other got 1/3. + - Increased the maximum length of an error message from 256 to 1024 + bytes. + - Added timestamp to debug log when a channel is closed, and remove + trailing newlines from debug log timestamps. + - Removed trailing newlines from SQL requests sent to the frontier + server (although not from the debug log). + - Cleaned out a duplicate internal implementation of strdup(). + +v2_7_7 - 14 February 2008 (dwd) + - Added a local client cache, for all queries that return a result + that is less than a specified size, default 10000 bytes. The + default size may be overridden with complex connection string + keyword "clientcachemaxresultsize=NNN" where NNN is the maximum + result size (in bytes) that will be cached. For typical current + CMSSW configurations this reduces the number of queries that are + sent to a server by an order of magnitude. + - Fixed bugs in the handling of the FrontierId when a grid + certificate has a subject that has illegal characters or is very + long, and added '@' to the list of allowed characters. + - Increased the maximum number of proxies from 4 to 16 and the + maximum number of servers from 6 to 16. + +v2_7_6 - 21 December 2007 (dwd) +This release is primarily about performance improvements, mainly + memory performance. This was done in a binary-compatible way. + - Eliminate one copy of each data item that was kept in AnyData, + and instead refer directly to the copy in memory that contains + all the data items in a query. + - Delete memory copies as soon as possible so there is never more + than two needed to be held allocated at once. + - Decode base64-encoded data as soon as it is read, saving 25% for + its space (especially significant for non-zipped queries). + - Keep the original data read from the frontier server in 64KB + chunks rather than repeatedly doubling the allocated space + and copying the data over (again, especially significant for + non-zipped queries). + - Replace the base64 decoding function with a faster one. + +v2_7_5 - 11 December 2007 (dwd) + - Fixed a debug log buffer overflow for long messages that has caused + corrupted log messages and crashes for long queries. Doubled the + buffer size, and made it safely truncate if a message is too long. + - When there are multiple addresses for a server name (round-robin + DNS), frontier client now cycles through the servers each time a + new connection is opened. + - Improved load balancing by dropping persistent connections after + every large (>16k) query. Some versions of squid were already + inconsistently dropping connections after large queries which + made load balancing much worse. + - Added options "loadbalance=proxies" and "loadbalance=servers" to + randomly select which proxy or server (respectively) to try first + rather than starting at the beginning of the list. + - Changed the retry strategy to this: + 1. Whenever a new connection is opened, reset the proxies & + servers lists to start the retry order at the beginning. If + doing load balancing, also randomly select a new beginning + proxy or server. + 2. If an error occurs, and it is a protocol error, assume the + problem is with the server so go to step 4. Otherwise the + error is a network error, go to step 3. If doing load + balancing, also mark the erring proxy or server so it won't be + tried again until all the proxies or servers have had errors. + 3. If there are multiple proxies, try them all before attempting any + forced refreshes, and if all proxies fail then go back to the + begining of the proxy list and try refreshes on them all. + 4. If those all fail, then try all the servers in order, and if + those all fail then go back to the beginning of the server list + and try refreshes on all of them. + - Eliminated a compiler warning on big endian Mac (PowerPC). + - Included a timestamp in warning messages about connection failures. + - Added server address to socket error messages. + - The debug log now includes the full size of objects even when they + aren't compressed. + - When $FRONTIER_LOG_FILE is set and begins with "+", and + $FRONTIER_LOG_LEVEL is set to warning or debug, the "+" is now + removed from the log file name and warning messages get sent to + to stdout in addition to the log file. + - If $CMSSW_VERSION is set, it is now included in the "frontier-id" + header which is sent in queries so the server can log what version + of CMSSW is being used (if any). If $CMSSW_VERSION is not set, it + now just says "client" in that place before the frontier client + version number (it used to say "GCC"). + - Added a header to all source files except the few that are imported + from elsewhere which says that the source is covered under the + GNU LGPL. Details in COPYING. + +v2_7_4 - 26 June 2007 (dwd) + - Fixed portability of build of builtin expat; it wasn't working on + 64-bit Linux since version 2.7.3 because -fPIC gcc option wasn't + being used. Also upgraded builtin expat from version 1.95.8 to + 2.0.1. + +v2_7_3 - 25 May 2007 (dwd) + - Ported to Intel & PowerPC Mac. Tested on Mac OS X 10.4.9 and + Xcode 2.4.1 (gcc 4.0.1). + +v2_7_2 - 23 March 2007 (dwd) + - Added a feature to use the subject name from a grid certificate + found at $X509_USER_PROXY if it is present, instead of the user's + name from /etc/passwd. + - Added OPENSSL_DIR Makefile command line variable to specify a + non-default directory for the openssl package. + +v2_7_1 - 22 March 2007 (dwd) + - Changed frontier_client library user-requested reload from + immediate cache refresh to adding "&ttl=short" to the query URL. + This tells the Frontier servlet that the time-to-live of the cache + of the query should be a "short" time, as defined by the server. + This requires servlet version 3.5 to work properly; older servlet + versions will still work but everything will expire at the same + time. + - Added complex connection string option "forcereload" and + environment variable $FRONTIER_FORCERELOAD. A value of "short" + in either forces an immediate cache refresh on the short time-to- + live queries and a value of "long" forces a refresh on all queries. + The default is "none". + +v2_7_0 - 4 January 2007 (dwd) + Note that this release is not binary compatible, although it is + source compatible. + - Dropped the _cms designation from version numbers. + - Changed AnyData::assignString() to avoid a new and delete for + non-"array" data types. This is now preferred for client programs + to use in place of AnyData::getString() because it avoids a new and + delete for all data types. + - Added a private AnyData::getStrBuf() function to the C++ API to get + access to a re-usable buffer for array types, to avoid having to do + a new and delete for every piece of data that's of type array. + This is only intended to be used internally by the "friend" class + Session. + - Added support for persistent connections that can be re-used for + multiple queries. The C++ API is to use two new C++ classes called + Connection and Session instead of the previous DataSource (although + that still works for backward source compatibility with one object + per query). Create one Connection object to hold connections to + the frontier proxies & servers for a long time and create one + Session object for each query. Sometimes (when there's too much + time between queries and when query results are large) the actual + TCP connection does get dropped by the squid proxy, but it is + automatically reconnected, transparent to the C++ API. + - Keep track of sequence numbers on channels and on queries within a + channel for debugging purposes. + - Move messages that report connection problems and retries from the + debug FRONTIER_LOG_LEVEL to the warning FRONTIER_LOG_LEVEL. + - Changed the default connect timeout from 3 seconds to 5 seconds. + - Added the ability to change the connect, read, and write timeout + seconds via environment variables FRONTIER_CONNECTTIMEOUTSECS, + FRONTIER_READTIMEOUTSECS, and FRONTIER_WRITETIMEOUTSECS or via + the complex connection string entries connecttimeoutsecs, + readtimeoutsecs, or writetimeoutsecs. + - Re-try selects if they fail with EINTR (which can happen when + profiling). + - Added --retrieve-zip-level to fnget.py. + - Added fnget2.py which is like fnget.py only supports multiple + queries per connection. + +v2_6_0_cms 09/18/2006 (dwd) + - Change only the version number because the binary API changed in + an incompatible way in v2_5_2_cms and that means a new second-level + version number should be started. + +v2_5_2_cms 09/13/2006 (dwd) + - Add debug messages for pre-encoded query, the beginning of the + post-decoded result, and the number of keepalives received (if any). + - Make the destructor of the FrontierException class be virtual + because Radovan Chytracek said it was causing stack corruption + when an exception occurred. + +v2_5_1_cms 08/07/2006 (dwd) + - Instead of overloading $FRONTIER_SERVER to mean both a list of + servers and the physical servers that correspond to the logical + server name, add environment variable $FRONTIER_PHYSICALSERVERS to + mean the latter. If the DataSource::setDefaultParams API is not + used, its values now can come from FRONTIER_LOGICALSERVER and + FRONTIER_PHYSICALSERVERS. Also, instead of ignoring the logical + server name when it is seen, the physical servers are now + substituted in its place (which is a little more intuitive). + - Added new keyword for complex parenthesized connection strings + called "logicalserverurl". If it is defined, its value becomes the + logical server for later connections and the rest of the string + becomes the physical servers. This was added because CORAL/POOL + uses one connection string for the first few frontier client + connections and then looks up subsequent connection strings in the + POOL file catalog. I didn't want to have to have the long + connection strings in both places resulting in difficulty with + keeping them all the same, and CMS did not want to pass them in + environment variables, so I proposed this solution. + +v2_5_0_cms 07/26/2006 (dwd) + - Make uncompressing of zipped payload work with 64 bit compiler + (where a long is 8 bytes and int is still 4 bytes). + - Add notion of a "logical server" which is a server name that gets + ignored in favor of servers/proxies that have been set by other + means. + - Add new C++ API class function DataSource::setDefaultParams which + takes parameters of a logical server name and a "parameter list" + which can be either a complex server connection string (see note + from v2_4_6_cms) or just a simple server string. If this API is + not used, the values can come instead from optional environment + variables FRONTIER_LOGICALSERVER and FRONTIER_SERVER respectively. + - Added new optional environment variable FRONTIER_RETRIEVEZIPLEVEL + to set the retrieved zip level if it isn't set any other way. + +v2_4_7_cms 07/12/2006 (dwd) + - Fixed bug that limited the maximum number of servers and proxies to + 3 instead of the intended 4. Also, increased the maximum number of + servers to 6. Removed the code that eliminated duplicate servers + and proxies because we may want to deliberately include duplicates + as part of a retry strategy. + +v2_4_6_cms 07/11/2006 (dwd) + - Added support for handling complex server connection strings + containing parenthesized keyword-value pairs. This was added to + make it straightforward for higher CMS software levels to pass a + single string to specify one connection. The format is + (keyword=value)... where keyword is one of serverurl, proxyurl, or + retrieve-ziplevel. Keywords can appear multiple times, and + anything between parenthesized components is ignored. + - Reduced network timeouts when talking to frontier server: + - connect from 30 to 3 seconds - initial connect should be quick + and don't want to wait too long in case a server is down + - writes from 30 to 5 seconds - writes should be quick unless + queues are large, which shouldn't happen + - reads from 30 to 10 seconds - server now sends keepalives + every 5 seconds if it isn't doing anything + - Added support of ZLIB_DIR for the make command line to point to an + external zlib package different than the standard zlib + +v2_4_5_cms 06/16/2006 (dwd) + - prevent the attempted build of the internal expat from deleting + $(EXPAT_DIR) if it is passed on the make command line, and from + installing the internal expat into the external $(EXPAT_DIR). + - add showing the server version in the debug log + +v2_4_4_cms 06/07/2006 (dwd) + - added optional zipping of the retrieved payload. The C++ API is + Request::setRetrieveZipLevel(level) where 0 is off, 1 is fastest, + 5 is normal, and 9 is best compression. Default is 5. Requires + corresponding update in frontier servlet, version 3.1 or greater. + - eliminated another header file compile problem in FrontierException.hpp + just like the one recently fixed in frontier-cpp.h, and eliminated + a compile error seen when the inline LogicError constructor from + the same file is used + - eliminated some compiler warnings in frontier-cpp.h + +v2_4_3_cms 06/01/2006 (dwd) + - fixed a compile problem seen on amd64 in frontier-cpp.h + - added frontier_initdebug C function that accepts a debug log file name + and log level, and a frontier::init overloaded function that accepts + the same two parameters + +v2_4_2_cms 12/25/2005 (sv) + - support for configuring client with multiple servers/proxies + - initial work on exceptions for the C++ api + +v2_4_1_cms 10/25/2005 (sv) + - distribution changes: if external $EXPAT_DIR is provided, the expat + files will not be included in the frontier library; however, client + application will have to link against external libraries in $EXPAT_DIR/lib + +v2_4_0_cms 10/12/2005 (sv) + - cms pool clients seem to be working + - fn-any, fn-req are not working against service on edge due to interface + change relative to v2_3_1 release + +v2_1_2 01/20/2005 + - bugs in NULL strings fixed + - diagnostic message from server now included into error message + - fn-any now shows strings in single quotes. Quotes and backslashes inside + strings are C-style escaped. + - for NULL strings fn-any pronts NULL (without quotes). +Sergey Kosyakov, serge@fnal.gov + +v2_1_1 01/10/2005 + - bug in NULL fields support fixed + - compiles with GCC-3.4 +Sergey Kosyakov, serge@fnal.gov + +v2.0.2 10/25/2004 + - bug which could destroy failover chain has been fixed + - library version changed to 2.0.2 +Sergey Kosyakov, serge@fnal.gov + +v2.0.1 10/25/2004 + - MetaRequest class added; it requests column names and types information + - test-any.cc (fn-any) rewritten to use MetaRequest (example of usage) + - client passes process id, user id, user name and user full name (if set) in requests to server + - library version excavated to 2.0.1 +Sergey Kosyakov, serge@fnal.gov + +v1.1.2 09/17/2004 +- frontier.h and frontier-cpp.h moved into include/frontier_client for consistency +- new method DataSource::isEOF() allows to verify full response demarshalling +- two new methods for diag/benchmarking tools: DataSource:getRSBinarySize() and DataSource::getRSBinaryPos() +- fn-any adapted to test isEOF() +Sergey Kosyakov, serge@fnal.gov + +v1.1.1 (GCC only) 09/15/2004 +- compiled with CDF KIT gcc-3.3.1 +Sergey Kosyakov, serge@fnal.gov + +v1.1 09/14/2004 +- Per-thread (global if not threaded) error message holder +- uniform error handling with detailed messages +- number of error codes reduced to 7l they are logically combined in 4 groups +- the library never calls abort() any more +- attempt to refresh cache before server failover +- uniform log report +- 4 log levels: nolog, error, warning and debug +- logs could be redirected to file (stdout by default) +- 2 new environment variables to control log: FRONTIER_LOG_LEVEL and FRONTIER_LOG_FILE +- automatic kit preparation when "make dist" +- 4 executable into the KIT: fn-maincc, fn-maintest, fn-pescalib and fn-any +- fn-any can show any XSD object as it is returned by the server +- AnyData optimized by inlining and calls streamlining +- new public call in DataSource: getAnyData(AnyData *any) +- new public call in AnyData: isEOR() +- "make all" (or just "make") now builds executables as well +- EOR is fully supported now (see test-any.cc as an usage example) +- the lib version elevated to 1.1.0 +Sergey Kosyakov, serge@fnal.gov + +v1.0.2 +LD_LIBRARY_PATH bug fixed +Sergey Kosyakov, serge@fnal.gov + +v1.0.1 +The library name has been changed to libfrontier_client.so +Sergey Kosyakov, serge@fnal.gov + +v1.0 +This is the first release of Frontier client API. +Sergey Kosyakov, serge@fnal.gov diff --git a/anydata.cc b/anydata.cc new file mode 100644 index 0000000..d019e42 --- /dev/null +++ b/anydata.cc @@ -0,0 +1,167 @@ +/* + * frontier client C++ API AnyData class + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#include +#include +#include + +using namespace frontier; + +static const char *blob_type_name[]={"byte","int4","int8","float","double","time","string","EOR"}; + +const char *frontier::getFieldTypeName(BLOB_TYPE t) + { + t=t&(~BLOB_BIT_NULL); + return blob_type_name[t]; + } + +int AnyData::castToInt() + { + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"converting %s to int",blob_type_name[t]); + switch(t) + { + case BLOB_TYPE_BYTE: return (int)v.b; + case BLOB_TYPE_TIME: + case BLOB_TYPE_INT8: return (int)v.i8; + case BLOB_TYPE_FLOAT: return (int)v.f; + case BLOB_TYPE_DOUBLE: return (int)v.d; + case BLOB_TYPE_ARRAY_BYTE: return atoi(v.str.p); + default: break; // Just to make this geek gcc happy + } + type_error=FRONTIER_EUNKNOWN; + frontier_setErrorMsg(__FILE__,__LINE__,"something wrong out there"); + return 0; + } + +long long AnyData::castToLongLong() + { + if(t!=BLOB_TYPE_INT4) frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"converting %s to long long",blob_type_name[t]); + switch(t) + { + case BLOB_TYPE_BYTE: return (long long)v.b; + case BLOB_TYPE_INT4: return (long long)v.i4; + case BLOB_TYPE_FLOAT: return (long long)v.f; + case BLOB_TYPE_DOUBLE: return (long long)v.d; + case BLOB_TYPE_ARRAY_BYTE: return atoll(v.str.p); + default: break; // Just to make this geek gcc happy + } + type_error=FRONTIER_EUNKNOWN; + frontier_setErrorMsg(__FILE__,__LINE__,"something wrong out there"); + return 0; + } + +float AnyData::castToFloat() + { + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"converting %s to float",blob_type_name[t]); + switch(t) + { + case BLOB_TYPE_BYTE: return (float)v.b; + case BLOB_TYPE_INT4: return (float)v.i4; + case BLOB_TYPE_TIME: + case BLOB_TYPE_INT8: return (float)v.i8; + case BLOB_TYPE_DOUBLE: return (float)v.d; + case BLOB_TYPE_ARRAY_BYTE: return (float)atof(v.str.p); + default: break; // Just to make this geek gcc happy + } + type_error=FRONTIER_EUNKNOWN; + frontier_setErrorMsg(__FILE__,__LINE__,"something wrong out there"); + return 0; + } + +double AnyData::castToDouble() + { + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"converting %s to double",blob_type_name[t]); + switch(t) + { + case BLOB_TYPE_BYTE: return (double)v.b; + case BLOB_TYPE_INT4: return (double)v.i4; + case BLOB_TYPE_TIME: + case BLOB_TYPE_INT8: return (double)v.i8; + case BLOB_TYPE_FLOAT: return (double)v.f; + case BLOB_TYPE_ARRAY_BYTE: return atof(v.str.p); + default: break; // Just to make this geek gcc happy + } + type_error=FRONTIER_EUNKNOWN; + frontier_setErrorMsg(__FILE__,__LINE__,"something wrong out there"); + return 0; + } + +std::string* AnyData::castToString() + { + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"converting %s to string",blob_type_name[t]); + std::ostringstream oss; + switch(t) + { + case BLOB_TYPE_BYTE: oss<assign(v.str.p,v.str.s); + return; + } + + std::ostringstream oss; + switch(t) + { + case BLOB_TYPE_BYTE: oss<assign(oss.str()); + } + +void AnyData::clean() + { + // the cleanup work is now done in the Session instead + if(sessionp)sessionp->clean(); + sessionp=NULL; + } + +AnyData::~AnyData() + { + clean(); + } diff --git a/chashtable/Makefile b/chashtable/Makefile new file mode 100644 index 0000000..3b7b5e9 --- /dev/null +++ b/chashtable/Makefile @@ -0,0 +1,26 @@ + +tester: hashtable.o tester.o hashtable_itr.o + gcc -g -Wall -O -lm -o tester hashtable.o hashtable_itr.o tester.o + +all: tester old_tester + +tester.o: tester.c + gcc -g -Wall -O -c tester.c -o tester.o + +old_tester: hashtable_powers.o tester.o hashtable_itr.o + gcc -g -Wall -O -o old_tester hashtable_powers.o hashtable_itr.o tester.o + +hashtable_powers.o: hashtable_powers.c + gcc -g -Wall -O -c hashtable_powers.c -o hashtable_powers.o + +hashtable.o: hashtable.c + gcc -g -Wall -O -c hashtable.c -o hashtable.o + +hashtable_itr.o: hashtable_itr.c + gcc -g -Wall -O -c hashtable_itr.c -o hashtable_itr.o + +tidy: + rm *.o + +clean: tidy + rm -f tester old_tester diff --git a/chashtable/README b/chashtable/README new file mode 100644 index 0000000..602a676 --- /dev/null +++ b/chashtable/README @@ -0,0 +1,2 @@ +This package came from http://www.cl.cam.ac.uk/~cwc22/hashtable/ +and is covered by the new BSD license. diff --git a/chashtable/hashtable.c b/chashtable/hashtable.c new file mode 100644 index 0000000..0af1fc0 --- /dev/null +++ b/chashtable/hashtable.c @@ -0,0 +1,301 @@ +/* Copyright (C) 2004 Christopher Clark */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include +#include +#include +#include + +/* +Credit for primes table: Aaron Krowne + http://br.endernet.org/~akrowne/ + http://planetmath.org/encyclopedia/GoodHashTablePrimes.html +*/ +static const unsigned int primes[] = { +53, 97, 193, 389, +769, 1543, 3079, 6151, +12289, 24593, 49157, 98317, +196613, 393241, 786433, 1572869, +3145739, 6291469, 12582917, 25165843, +50331653, 100663319, 201326611, 402653189, +805306457, 1610612741 +}; +const unsigned int prime_table_length = sizeof(primes)/sizeof(primes[0]); +const float max_load_factor = 0.65; + +/*****************************************************************************/ +struct hashtable * +create_hashtable(unsigned int minsize, + unsigned int (*hashf) (void*), + int (*eqf) (void*,void*)) +{ + struct hashtable *h; + unsigned int pindex, size = primes[0]; + /* Check requested hashtable isn't too large */ + if (minsize > (1u << 30)) return NULL; + /* Enforce size as prime */ + for (pindex=0; pindex < prime_table_length; pindex++) { + if (primes[pindex] > minsize) { size = primes[pindex]; break; } + } + h = (struct hashtable *)malloc(sizeof(struct hashtable)); + if (NULL == h) return NULL; /*oom*/ + h->table = (struct entry **)malloc(sizeof(struct entry*) * size); + if (NULL == h->table) { free(h); return NULL; } /*oom*/ + memset(h->table, 0, size * sizeof(struct entry *)); + h->tablelength = size; + h->primeindex = pindex; + h->entrycount = 0; + h->hashfn = hashf; + h->eqfn = eqf; + h->loadlimit = (unsigned int) ceil(size * max_load_factor); + return h; +} + +/*****************************************************************************/ +unsigned int +hashtable_hash(struct hashtable *h, void *k) +{ + /* Aim to protect against poor hash functions by adding logic here + * - logic taken from java 1.4 hashtable source */ + unsigned int i = h->hashfn(k); + i += ~(i << 9); + i ^= ((i >> 14) | (i << 18)); /* >>> */ + i += (i << 4); + i ^= ((i >> 10) | (i << 22)); /* >>> */ + return i; +} + +/*****************************************************************************/ +static int +hashtable_expand(struct hashtable *h) +{ + /* Double the size of the table to accomodate more entries */ + struct entry **newtable; + struct entry *e; + struct entry **pE; + unsigned int newsize, i, index; + /* Check we're not hitting max capacity */ + if (h->primeindex == (prime_table_length - 1)) return 0; + newsize = primes[++(h->primeindex)]; + + newtable = (struct entry **)malloc(sizeof(struct entry*) * newsize); + if (NULL != newtable) + { + memset(newtable, 0, newsize * sizeof(struct entry *)); + /* This algorithm is not 'stable'. ie. it reverses the list + * when it transfers entries between the tables */ + for (i = 0; i < h->tablelength; i++) { + while (NULL != (e = h->table[i])) { + h->table[i] = e->next; + index = indexFor(newsize,e->h); + e->next = newtable[index]; + newtable[index] = e; + } + } + free(h->table); + h->table = newtable; + } + /* Plan B: realloc instead */ + else + { + newtable = (struct entry **) + realloc(h->table, newsize * sizeof(struct entry *)); + if (NULL == newtable) { (h->primeindex)--; return 0; } + h->table = newtable; + memset(newtable[h->tablelength], 0, newsize - h->tablelength); + for (i = 0; i < h->tablelength; i++) { + for (pE = &(newtable[i]), e = *pE; e != NULL; e = *pE) { + index = indexFor(newsize,e->h); + if (index == i) + { + pE = &(e->next); + } + else + { + *pE = e->next; + e->next = newtable[index]; + newtable[index] = e; + } + } + } + } + h->tablelength = newsize; + h->loadlimit = (unsigned int) ceil(newsize * max_load_factor); + return -1; +} + +/*****************************************************************************/ +unsigned int +hashtable_count(struct hashtable *h) +{ + return h->entrycount; +} + +/*****************************************************************************/ +int +hashtable_insert(struct hashtable *h, void *k, void *v) +{ + /* This method allows duplicate keys - but they shouldn't be used */ + unsigned int index; + struct entry *e; + if (++(h->entrycount) > h->loadlimit) + { + /* Ignore the return value. If expand fails, we should + * still try cramming just this value into the existing table + * -- we may not have memory for a larger table, but one more + * element may be ok. Next time we insert, we'll try expanding again.*/ + hashtable_expand(h); + } + e = (struct entry *)malloc(sizeof(struct entry)); + if (NULL == e) { --(h->entrycount); return 0; } /*oom*/ + e->h = hashtable_hash(h,k); + index = indexFor(h->tablelength,e->h); + e->k = k; + e->v = v; + e->next = h->table[index]; + h->table[index] = e; + return -1; +} + +/*****************************************************************************/ +void * /* returns value associated with key */ +hashtable_search(struct hashtable *h, void *k) +{ + struct entry *e; + unsigned int hashvalue, index; + hashvalue = hashtable_hash(h,k); + index = indexFor(h->tablelength,hashvalue); + e = h->table[index]; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) return e->v; + e = e->next; + } + return NULL; +} + +/*****************************************************************************/ +void * /* returns value associated with key */ +hashtable_remove(struct hashtable *h, void *k) +{ + /* TODO: consider compacting the table when the load factor drops enough, + * or provide a 'compact' method. */ + + struct entry *e; + struct entry **pE; + void *v; + unsigned int hashvalue, index; + + hashvalue = hashtable_hash(h,k); + index = indexFor(h->tablelength,hashvalue); + pE = &(h->table[index]); + e = *pE; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + *pE = e->next; + h->entrycount--; + v = e->v; + freekey(e->k); + free(e); + return v; + } + pE = &(e->next); + e = e->next; + } + return NULL; +} + +/*****************************************************************************/ +/* destroy */ +void +hashtable_destroy(struct hashtable *h, int free_values) +{ + unsigned int i; + struct entry *e, *f; + struct entry **table = h->table; + if (free_values) + { + for (i = 0; i < h->tablelength; i++) + { + e = table[i]; + while (NULL != e) + { f = e; e = e->next; freekey(f->k); free(f->v); free(f); } + } + } + else + { + for (i = 0; i < h->tablelength; i++) + { + e = table[i]; + while (NULL != e) + { f = e; e = e->next; freekey(f->k); free(f); } + } + } + free(h->table); + free(h); +} + +/*****************************************************************************/ +/* stats */ +void +hashtable_stats(struct hashtable *h, char *statbuf, int maxstatlen) +{ + unsigned int i; + struct entry *e; + struct entry **table = h->table; + int n; + int empty = 0, max = 0, total = 0; + for (i = 0; i < h->tablelength; i++) + { + e = table[i]; + if (!e) empty++; + n = 0; + while (NULL != e) + { + n++; + total++; + e = e->next; + } + if (n > max) max = n; + } + snprintf(statbuf, maxstatlen-1,"buckets: %d, empty: %d, max: %d, total: %d", + h->tablelength, empty, max, total); +} + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/chashtable/hashtable.h b/chashtable/hashtable.h new file mode 100644 index 0000000..09e19e5 --- /dev/null +++ b/chashtable/hashtable.h @@ -0,0 +1,211 @@ +/* Copyright (C) 2002 Christopher Clark */ + +#ifndef __HASHTABLE_CWC22_H__ +#define __HASHTABLE_CWC22_H__ + +struct hashtable; + +/* Example of use: + * + * struct hashtable *h; + * struct some_key *k; + * struct some_value *v; + * + * static unsigned int hash_from_key_fn( void *k ); + * static int keys_equal_fn ( void *key1, void *key2 ); + * + * h = create_hashtable(16, hash_from_key_fn, keys_equal_fn); + * k = (struct some_key *) malloc(sizeof(struct some_key)); + * v = (struct some_value *) malloc(sizeof(struct some_value)); + * + * (initialise k and v to suitable values) + * + * if (! hashtable_insert(h,k,v) ) + * { exit(-1); } + * + * if (NULL == (found = hashtable_search(h,k) )) + * { printf("not found!"); } + * + * if (NULL == (found = hashtable_remove(h,k) )) + * { printf("Not found\n"); } + * + */ + +/* Macros may be used to define type-safe(r) hashtable access functions, with + * methods specialized to take known key and value types as parameters. + * + * Example: + * + * Insert this at the start of your file: + * + * DEFINE_HASHTABLE_INSERT(insert_some, struct some_key, struct some_value); + * DEFINE_HASHTABLE_SEARCH(search_some, struct some_key, struct some_value); + * DEFINE_HASHTABLE_REMOVE(remove_some, struct some_key, struct some_value); + * + * This defines the functions 'insert_some', 'search_some' and 'remove_some'. + * These operate just like hashtable_insert etc., with the same parameters, + * but their function signatures have 'struct some_key *' rather than + * 'void *', and hence can generate compile time errors if your program is + * supplying incorrect data as a key (and similarly for value). + * + * Note that the hash and key equality functions passed to create_hashtable + * still take 'void *' parameters instead of 'some key *'. This shouldn't be + * a difficult issue as they're only defined and passed once, and the other + * functions will ensure that only valid keys are supplied to them. + * + * The cost for this checking is increased code size and runtime overhead + * - if performance is important, it may be worth switching back to the + * unsafe methods once your program has been debugged with the safe methods. + * This just requires switching to some simple alternative defines - eg: + * #define insert_some hashtable_insert + * + */ + +/***************************************************************************** + * create_hashtable + + * @name create_hashtable + * @param minsize minimum initial size of hashtable + * @param hashfunction function for hashing keys + * @param key_eq_fn function for determining key equality + * @return newly created hashtable or NULL on failure + */ + +struct hashtable * +create_hashtable(unsigned int minsize, + unsigned int (*hashfunction) (void*), + int (*key_eq_fn) (void*,void*)); + +/***************************************************************************** + * hashtable_insert + + * @name hashtable_insert + * @param h the hashtable to insert into + * @param k the key - hashtable claims ownership and will free on removal + * @param v the value - does not claim ownership + * @return non-zero for successful insertion + * + * This function will cause the table to expand if the insertion would take + * the ratio of entries to table size over the maximum load factor. + * + * This function does not check for repeated insertions with a duplicate key. + * The value returned when using a duplicate key is undefined -- when + * the hashtable changes size, the order of retrieval of duplicate key + * entries is reversed. + * If in doubt, remove before insert. + */ + +int +hashtable_insert(struct hashtable *h, void *k, void *v); + +#define DEFINE_HASHTABLE_INSERT(fnname, keytype, valuetype) \ +int fnname (struct hashtable *h, keytype *k, valuetype *v) \ +{ \ + return hashtable_insert(h,k,v); \ +} + +/***************************************************************************** + * hashtable_search + + * @name hashtable_search + * @param h the hashtable to search + * @param k the key to search for - does not claim ownership + * @return the value associated with the key, or NULL if none found + */ + +void * +hashtable_search(struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_SEARCH(fnname, keytype, valuetype) \ +valuetype * fnname (struct hashtable *h, keytype *k) \ +{ \ + return (valuetype *) (hashtable_search(h,k)); \ +} + +/***************************************************************************** + * hashtable_remove + + * @name hashtable_remove + * @param h the hashtable to remove the item from + * @param k the key to search for - does not claim ownership + * @return the value associated with the key, or NULL if none found + */ + +void * /* returns value */ +hashtable_remove(struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_REMOVE(fnname, keytype, valuetype) \ +valuetype * fnname (struct hashtable *h, keytype *k) \ +{ \ + return (valuetype *) (hashtable_remove(h,k)); \ +} + + +/***************************************************************************** + * hashtable_count + + * @name hashtable_count + * @param h the hashtable + * @return the number of items stored in the hashtable + */ +unsigned int +hashtable_count(struct hashtable *h); + + +/***************************************************************************** + * hashtable_destroy + + * @name hashtable_destroy + * @param h the hashtable + * @param free_values whether to call 'free' on the remaining values + */ + +void +hashtable_destroy(struct hashtable *h, int free_values); + +/***************************************************************************** + * hashtable_stats + + * @name hashtable_stats + * @param h the hashtable + * @param statbuf buffer to print statistics to, suggest size 128 + * @param maxstatlen size of statbuf + */ + +void +hashtable_stats(struct hashtable *h, char *statbuf, int maxstatlen); + +#endif /* __HASHTABLE_CWC22_H__ */ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/chashtable/hashtable_itr.c b/chashtable/hashtable_itr.c new file mode 100644 index 0000000..902f2df --- /dev/null +++ b/chashtable/hashtable_itr.c @@ -0,0 +1,188 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include "hashtable_itr.h" +#include /* defines NULL */ + +/*****************************************************************************/ +/* hashtable_iterator - iterator constructor */ + +struct hashtable_itr * +hashtable_iterator(struct hashtable *h) +{ + unsigned int i, tablelength; + struct hashtable_itr *itr = (struct hashtable_itr *) + malloc(sizeof(struct hashtable_itr)); + if (NULL == itr) return NULL; + itr->h = h; + itr->e = NULL; + itr->parent = NULL; + tablelength = h->tablelength; + itr->index = tablelength; + if (0 == h->entrycount) return itr; + + for (i = 0; i < tablelength; i++) + { + if (NULL != h->table[i]) + { + itr->e = h->table[i]; + itr->index = i; + break; + } + } + return itr; +} + +/*****************************************************************************/ +/* key - return the key of the (key,value) pair at the current position */ +/* value - return the value of the (key,value) pair at the current position */ + +void * +hashtable_iterator_key(struct hashtable_itr *i) +{ return i->e->k; } + +void * +hashtable_iterator_value(struct hashtable_itr *i) +{ return i->e->v; } + +/*****************************************************************************/ +/* advance - advance the iterator to the next element + * returns zero if advanced to end of table */ + +int +hashtable_iterator_advance(struct hashtable_itr *itr) +{ + unsigned int j,tablelength; + struct entry **table; + struct entry *next; + if (NULL == itr->e) return 0; /* stupidity check */ + + next = itr->e->next; + if (NULL != next) + { + itr->parent = itr->e; + itr->e = next; + return -1; + } + tablelength = itr->h->tablelength; + itr->parent = NULL; + if (tablelength <= (j = ++(itr->index))) + { + itr->e = NULL; + return 0; + } + table = itr->h->table; + while (NULL == (next = table[j])) + { + if (++j >= tablelength) + { + itr->index = tablelength; + itr->e = NULL; + return 0; + } + } + itr->index = j; + itr->e = next; + return -1; +} + +/*****************************************************************************/ +/* remove - remove the entry at the current iterator position + * and advance the iterator, if there is a successive + * element. + * If you want the value, read it before you remove: + * beware memory leaks if you don't. + * Returns zero if end of iteration. */ + +int +hashtable_iterator_remove(struct hashtable_itr *itr) +{ + struct entry *remember_e, *remember_parent; + int ret; + + /* Do the removal */ + if (NULL == (itr->parent)) + { + /* element is head of a chain */ + itr->h->table[itr->index] = itr->e->next; + } else { + /* element is mid-chain */ + itr->parent->next = itr->e->next; + } + /* itr->e is now outside the hashtable */ + remember_e = itr->e; + itr->h->entrycount--; + freekey(remember_e->k); + + /* Advance the iterator, correcting the parent */ + remember_parent = itr->parent; + ret = hashtable_iterator_advance(itr); + if (itr->parent == remember_e) { itr->parent = remember_parent; } + free(remember_e); + return ret; +} + +/*****************************************************************************/ +int /* returns zero if not found */ +hashtable_iterator_search(struct hashtable_itr *itr, + struct hashtable *h, void *k) +{ + struct entry *e, *parent; + unsigned int hashvalue, index; + + hashvalue = hashtable_hash(h,k); + index = indexFor(h->tablelength,hashvalue); + + e = h->table[index]; + parent = NULL; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + itr->index = index; + itr->e = e; + itr->parent = parent; + itr->h = h; + return -1; + } + parent = e; + e = e->next; + } + return 0; +} + + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/chashtable/hashtable_itr.h b/chashtable/hashtable_itr.h new file mode 100644 index 0000000..eea699a --- /dev/null +++ b/chashtable/hashtable_itr.h @@ -0,0 +1,112 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#ifndef __HASHTABLE_ITR_CWC22__ +#define __HASHTABLE_ITR_CWC22__ +#include "hashtable.h" +#include "hashtable_private.h" /* needed to enable inlining */ + +/*****************************************************************************/ +/* This struct is only concrete here to allow the inlining of two of the + * accessor functions. */ +struct hashtable_itr +{ + struct hashtable *h; + struct entry *e; + struct entry *parent; + unsigned int index; +}; + + +/*****************************************************************************/ +/* hashtable_iterator + */ + +struct hashtable_itr * +hashtable_iterator(struct hashtable *h); + +/*****************************************************************************/ +/* hashtable_iterator_key + * - return the value of the (key,value) pair at the current position */ + +extern inline void * +hashtable_iterator_key(struct hashtable_itr *i) +{ + return i->e->k; +} + +/*****************************************************************************/ +/* value - return the value of the (key,value) pair at the current position */ + +extern inline void * +hashtable_iterator_value(struct hashtable_itr *i) +{ + return i->e->v; +} + +/*****************************************************************************/ +/* advance - advance the iterator to the next element + * returns zero if advanced to end of table */ + +int +hashtable_iterator_advance(struct hashtable_itr *itr); + +/*****************************************************************************/ +/* remove - remove current element and advance the iterator to the next element + * NB: if you need the value to free it, read it before + * removing. ie: beware memory leaks! + * returns zero if advanced to end of table */ + +int +hashtable_iterator_remove(struct hashtable_itr *itr); + +/*****************************************************************************/ +/* search - overwrite the supplied iterator, to point to the entry + * matching the supplied key. + h points to the hashtable to be searched. + * returns zero if not found. */ +int +hashtable_iterator_search(struct hashtable_itr *itr, + struct hashtable *h, void *k); + +#define DEFINE_HASHTABLE_ITERATOR_SEARCH(fnname, keytype) \ +int fnname (struct hashtable_itr *i, struct hashtable *h, keytype *k) \ +{ \ + return (hashtable_iterator_search(i,h,k)); \ +} + + + +#endif /* __HASHTABLE_ITR_CWC22__*/ + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/chashtable/hashtable_private.h b/chashtable/hashtable_private.h new file mode 100644 index 0000000..3e95f60 --- /dev/null +++ b/chashtable/hashtable_private.h @@ -0,0 +1,85 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#ifndef __HASHTABLE_PRIVATE_CWC22_H__ +#define __HASHTABLE_PRIVATE_CWC22_H__ + +#include "hashtable.h" + +/*****************************************************************************/ +struct entry +{ + void *k, *v; + unsigned int h; + struct entry *next; +}; + +struct hashtable { + unsigned int tablelength; + struct entry **table; + unsigned int entrycount; + unsigned int loadlimit; + unsigned int primeindex; + unsigned int (*hashfn) (void *k); + int (*eqfn) (void *k1, void *k2); +}; + +/*****************************************************************************/ +unsigned int +hash(struct hashtable *h, void *k); + +/*****************************************************************************/ +/* indexFor */ +static inline unsigned int +indexFor(unsigned int tablelength, unsigned int hashvalue) { + return (hashvalue % tablelength); +}; + +/* Only works if tablelength == 2^N */ +/*static inline unsigned int +indexFor(unsigned int tablelength, unsigned int hashvalue) +{ + return (hashvalue & (tablelength - 1u)); +} +*/ + +/*****************************************************************************/ +#define freekey(X) free(X) +/*define freekey(X) ; */ + + +/*****************************************************************************/ + +#endif /* __HASHTABLE_PRIVATE_CWC22_H__*/ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/chashtable/hashtable_utility.c b/chashtable/hashtable_utility.c new file mode 100644 index 0000000..f3aa7cf --- /dev/null +++ b/chashtable/hashtable_utility.c @@ -0,0 +1,71 @@ +/* Copyright (C) 2002 Christopher Clark */ + +#include "hashtable.h" +#include "hashtable_private.h" +#include "hashtable_utility.h" +#include +#include +#include + +/*****************************************************************************/ +/* hashtable_change + * + * function to change the value associated with a key, where there already + * exists a value bound to the key in the hashtable. + * Source due to Holger Schemel. + * + * */ +int +hashtable_change(struct hashtable *h, void *k, void *v) +{ + struct entry *e; + unsigned int hashvalue, index; + hashvalue = hashtable_hash(h,k); + index = indexFor(h->tablelength,hashvalue); + e = h->table[index]; + while (NULL != e) + { + /* Check hash value to short circuit heavier comparison */ + if ((hashvalue == e->h) && (h->eqfn(k, e->k))) + { + free(e->v); + e->v = v; + return -1; + } + e = e->next; + } + return 0; +} + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/chashtable/hashtable_utility.h b/chashtable/hashtable_utility.h new file mode 100644 index 0000000..56a0ffd --- /dev/null +++ b/chashtable/hashtable_utility.h @@ -0,0 +1,55 @@ +/* Copyright (C) 2002 Christopher Clark */ + +#ifndef __HASHTABLE_CWC22_UTILITY_H__ +#define __HASHTABLE_CWC22_UTILITY_H__ + +/***************************************************************************** + * hashtable_change + * + * function to change the value associated with a key, where there already + * exists a value bound to the key in the hashtable. + * Source due to Holger Schemel. + * + * @name hashtable_change + * @param h the hashtable + * @param key + * @param value + * + */ +int +hashtable_change(struct hashtable *h, void *k, void *v); + +#endif /* __HASHTABLE_CWC22_H__ */ + +/* + * Copyright (c) 2002, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/chashtable/tester.c b/chashtable/tester.c new file mode 100644 index 0000000..4678ffa --- /dev/null +++ b/chashtable/tester.c @@ -0,0 +1,270 @@ +/* Copyright (C) 2002, 2004 Christopher Clark */ + +#include "hashtable.h" +#include "hashtable_itr.h" +#include +#include +#include /* for memcmp */ + +static const int ITEM_COUNT = 4000; + +typedef unsigned int uint32_t; +typedef unsigned short uint16_t; + +/*****************************************************************************/ +struct key +{ + uint32_t one_ip; uint32_t two_ip; uint16_t one_port; uint16_t two_port; +}; + +struct value +{ + char *id; +}; + +DEFINE_HASHTABLE_INSERT(insert_some, struct key, struct value); +DEFINE_HASHTABLE_SEARCH(search_some, struct key, struct value); +DEFINE_HASHTABLE_REMOVE(remove_some, struct key, struct value); +DEFINE_HASHTABLE_ITERATOR_SEARCH(search_itr_some, struct key); + + +/*****************************************************************************/ +static unsigned int +hashfromkey(void *ky) +{ + struct key *k = (struct key *)ky; + return (((k->one_ip << 17) | (k->one_ip >> 15)) ^ k->two_ip) + + (k->one_port * 17) + (k->two_port * 13 * 29); +} + +static int +equalkeys(void *k1, void *k2) +{ + return (0 == memcmp(k1,k2,sizeof(struct key))); +} + +/*****************************************************************************/ +int +main(int argc, char **argv) +{ + struct key *k, *kk; + struct value *v, *found; + struct hashtable *h; + struct hashtable_itr *itr; + int i; + + h = create_hashtable(16, hashfromkey, equalkeys); + if (NULL == h) exit(-1); /*oom*/ + + +/*****************************************************************************/ +/* Insertion */ + for (i = 0; i < ITEM_COUNT; i++) + { + k = (struct key *)malloc(sizeof(struct key)); + if (NULL == k) { + printf("ran out of memory allocating a key\n"); + return 1; + } + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + v = (struct value *)malloc(sizeof(struct value)); + v->id = "a value"; + + if (!insert_some(h,k,v)) exit(-1); /*oom*/ + } + printf("After insertion, hashtable contains %u items.\n", + hashtable_count(h)); + +/*****************************************************************************/ +/* Hashtable search */ + k = (struct key *)malloc(sizeof(struct key)); + if (NULL == k) { + printf("ran out of memory allocating a key\n"); + return 1; + } + + for (i = 0; i < ITEM_COUNT; i++) + { + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + if (NULL == (found = search_some(h,k))) { + printf("BUG: key not found\n"); + } + } + +/*****************************************************************************/ +/* Hashtable iteration */ + /* Iterator constructor only returns a valid iterator if + * the hashtable is not empty */ + itr = hashtable_iterator(h); + i = 0; + if (hashtable_count(h) > 0) + { + do { + kk = hashtable_iterator_key(itr); + v = hashtable_iterator_value(itr); + /* here (kk,v) are a valid (key, value) pair */ + /* We could call 'hashtable_remove(h,kk)' - and this operation + * 'free's kk. However, the iterator is then broken. + * This is why hashtable_iterator_remove exists - see below. + */ + i++; + + } while (hashtable_iterator_advance(itr)); + } + printf("Iterated through %u entries.\n", i); + +/*****************************************************************************/ +/* Hashtable iterator search */ + + /* Try the search some method */ + for (i = 0; i < ITEM_COUNT; i++) + { + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + if (0 == search_itr_some(itr,h,k)) { + printf("BUG: key not found searching with iterator"); + } + } + +/*****************************************************************************/ +/* Hashtable removal */ + + for (i = 0; i < ITEM_COUNT; i++) + { + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + if (NULL == (found = remove_some(h,k))) { + printf("BUG: key not found for removal\n"); + } + } + printf("After removal, hashtable contains %u items.\n", + hashtable_count(h)); + +/*****************************************************************************/ +/* Hashtable destroy and create */ + + hashtable_destroy(h, 1); + h = NULL; + free(k); + + h = create_hashtable(160, hashfromkey, equalkeys); + if (NULL == h) { + printf("out of memory allocating second hashtable\n"); + return 1; + } + +/*****************************************************************************/ +/* Hashtable insertion */ + + for (i = 0; i < ITEM_COUNT; i++) + { + k = (struct key *)malloc(sizeof(struct key)); + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + v = (struct value *)malloc(sizeof(struct value)); + v->id = "a value"; + + if (!insert_some(h,k,v)) + { + printf("out of memory inserting into second hashtable\n"); + return 1; + } + } + printf("After insertion, hashtable contains %u items.\n", + hashtable_count(h)); + +/*****************************************************************************/ +/* Hashtable iterator search and iterator remove */ + + k = (struct key *)malloc(sizeof(struct key)); + if (NULL == k) { + printf("ran out of memory allocating a key\n"); + return 1; + } + + for (i = ITEM_COUNT - 1; i >= 0; i = i - 7) + { + k->one_ip = 0xcfccee40 + i; + k->two_ip = 0xcf0cee67 - (5 * i); + k->one_port = 22 + (7 * i); + k->two_port = 5522 - (3 * i); + + if (0 == search_itr_some(itr, h, k)) { + printf("BUG: key %u not found for search preremoval using iterator\n", i); + return 1; + } + if (0 == hashtable_iterator_remove(itr)) { + printf("BUG: key not found for removal using iterator\n"); + return 1; + } + } + free(itr); + +/*****************************************************************************/ +/* Hashtable iterator remove and advance */ + + for (itr = hashtable_iterator(h); + hashtable_iterator_remove(itr) != 0; ) { + ; + } + free(itr); + printf("After removal, hashtable contains %u items.\n", + hashtable_count(h)); + +/*****************************************************************************/ +/* Hashtable destroy */ + + hashtable_destroy(h, 1); + free(k); + return 0; +} + +/* + * Copyright (c) 2002, 2004, Christopher Clark + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the original author; nor the names of any contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/fn-base64.c b/fn-base64.c new file mode 100644 index 0000000..12689dd --- /dev/null +++ b/fn-base64.c @@ -0,0 +1,194 @@ +/* +** The frontier base64 decoder is based on source code that came without a +** copyright from http://www.jeremie.com/frolic/base64/, decode-1.c +** +** The frontier base64 encoder has the following copyright: +** Jack Jansen, CWI, July 1995. +** Brandon Long, September 2001. +** Python 2.3.3 code adapted by Sergey Kosyakov, May 2004 +** See http://www.python.org for license information +** +*/ + +#include "fn-base64.h" + +static unsigned char table_b2a_base64[] = +"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +#define BASE64_PAD '=' + +// These two below are for URL-compatible encoding +static unsigned char table_b2a_base64URL[] = +"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789.-"; +#define BASE64URL_PAD '_' + + +/* Max binary chunk size; limited only by available memory */ +#define BASE64_MAXBIN (1024*1024*1024) + +static int base64_table_a2b[256] = { + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, /* 00-0F */ + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, /* 10-1F */ + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,62,-1,-1,-1,63, /* 20-2F */ + 52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-1,-1,-1, /* 30-3F */ + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14, /* 40-4F */ + 15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1, /* 50-5F */ + -1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40, /* 60-6F */ + 41,42,43,44,45,46,47,48,49,50,51,-1,-1,-1,-1,-1, /* 70-7F */ + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, /* 80-8F */ + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, /* 90-9F */ + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, /* A0-AF */ + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, /* B0-BF */ + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, /* C0-CF */ + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, /* D0-DF */ + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, /* E0-EF */ + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 /* F0-FF */ +}; + +void fn_base64_stream_ascii2bin(fn_b64a2b_context *ctxt, + const unsigned char *ascii_data,int *ascii_lenp, + unsigned char *bin_data,int *bin_lenp) +{ + unsigned char *p; + const unsigned char *endp, *enda; + int d, dlast, phase; + unsigned char c; + + dlast = ctxt->dlast; + phase = ctxt->phase; + d = 0; + p = bin_data; + endp = p + *bin_lenp; + enda = ascii_data + *ascii_lenp; + while((ascii_data != enda) && (p != endp)) + { + d = base64_table_a2b[(int)(*ascii_data++)]; + if(d != -1) + { + switch(phase) + { + case 0: + ++phase; + break; + case 1: + c = ((dlast << 2) | ((d & 0x30) >> 4)); + *p++ = c; + ++phase; + break; + case 2: + c = (((dlast & 0xf) << 4) | ((d & 0x3c) >> 2)); + *p++ = c; + ++phase; + break; + case 3: + c = (((dlast & 0x03 ) << 6) | d); + *p++ = c; + phase = 0; + break; + } + dlast = d; + } + } + ctxt->dlast = dlast; + ctxt->phase = phase; + *ascii_lenp = enda - ascii_data; + *bin_lenp = endp - p; +} + +int fn_base64_ascii2bin(const unsigned char *ascii_data,int ascii_len, + unsigned char *bin_data,int bin_len) +{ + fn_b64a2b_context ctxt; + int save_bin_len=bin_len; + ctxt.dlast=ctxt.phase=0; + fn_base64_stream_ascii2bin(&ctxt,ascii_data,&ascii_len,bin_data,&bin_len); + return save_bin_len-bin_len; +} + +int fn_base64_bin2ascii(const unsigned char *bin_data,int bin_len, + unsigned char *ascii_data,int ascii_len) + { + int leftbits=0; + unsigned char this_ch; + unsigned int leftchar=0; + unsigned char *ptr=ascii_data; + + if(bin_len>BASE64_MAXBIN) return BASE64_NOSPACE; + + /* We're lazy and allocate too much (fixed up later). + "+3" leaves room for up to two pad characters and a trailing + newline. Note that 'b' gets encoded as 'Yg==\n' (1 in, 5 out). */ + if(bin_len*2+3>ascii_len) return BASE64_NOSPACE; + + for(;bin_len>0 ;bin_len--, bin_data++) + { + /* Shift the data into our buffer */ + leftchar = (leftchar << 8) | *bin_data; + leftbits += 8; + + /* See if there are 6-bit groups ready */ + while ( leftbits >= 6 ) { + this_ch = (leftchar >> (leftbits-6)) & 0x3f; + leftbits -= 6; + *ascii_data++ = table_b2a_base64[this_ch]; + } + } + if ( leftbits == 2 ) { + *ascii_data++ = table_b2a_base64[(leftchar&3) << 4]; + *ascii_data++ = BASE64_PAD; + *ascii_data++ = BASE64_PAD; + } else if ( leftbits == 4 ) { + *ascii_data++ = table_b2a_base64[(leftchar&0xf) << 2]; + *ascii_data++ = BASE64_PAD; + } + *ascii_data++ = '\n'; /* Append a courtesy newline */ + + ascii_len=ascii_data-ptr; + + return ascii_len; +} + + +// This method is modified to create URL-compatible output +int fn_base64URL_bin2ascii(const unsigned char *bin_data,int bin_len, + unsigned char *ascii_data,int ascii_len) + { + int leftbits=0; + unsigned char this_ch; + unsigned int leftchar=0; + unsigned char *ptr=ascii_data; + + if(bin_len>BASE64_MAXBIN) return BASE64_NOSPACE; + + /* We're lazy and allocate too much (fixed up later). + "+3" leaves room for up to two pad characters and a trailing + newline. Note that 'b' gets encoded as 'Yg==\n' (1 in, 5 out). */ + if(bin_len*2+3>ascii_len) return BASE64_NOSPACE; + + for(;bin_len>0 ;bin_len--, bin_data++) + { + /* Shift the data into our buffer */ + leftchar = (leftchar << 8) | *bin_data; + leftbits += 8; + + /* See if there are 6-bit groups ready */ + while ( leftbits >= 6 ) { + this_ch = (leftchar >> (leftbits-6)) & 0x3f; + leftbits -= 6; + *ascii_data++ = table_b2a_base64URL[this_ch]; + } + } + if ( leftbits == 2 ) { + *ascii_data++ = table_b2a_base64URL[(leftchar&3) << 4]; + *ascii_data++ = BASE64URL_PAD; + *ascii_data++ = BASE64URL_PAD; + } else if ( leftbits == 4 ) { + *ascii_data++ = table_b2a_base64URL[(leftchar&0xf) << 2]; + *ascii_data++ = BASE64URL_PAD; + } + /* A courtesy newline is ommited */ + + ascii_len=ascii_data-ptr; + + return ascii_len; +} diff --git a/fn-base64.h b/fn-base64.h new file mode 100644 index 0000000..c855537 --- /dev/null +++ b/fn-base64.h @@ -0,0 +1,54 @@ +/* + * frontier client base64 encode/decode header + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#ifndef __H_FN_BASE64__ +#define __H_FN_BASE64__ + +#define BASE64_NOSPACE -1 +#define BASE64_INVPADDING -2 + +struct s_fn_b64a2b_context { + int dlast; + int phase; +}; +typedef struct s_fn_b64a2b_context fn_b64a2b_context; + +// decode a portion of a stream of data from base64 ascii to binary +// caller must ensure that the ctxt structure is initialized to zero +// When finished, *ascii_lenp will have the number of bytes left unused +// in the ascii data, and *bin_lenp will have the number of bytes +// left in the binary data. +void fn_base64_stream_ascii2bin(fn_b64a2b_context *ctxt, + const unsigned char *ascii_data,int *ascii_lenp, + unsigned char *bin_data,int *bin_lenp); + +// decode from base64 ascii to binary +// return the amount of binary data produced +int fn_base64_ascii2bin(const unsigned char *ascii_data,int ascii_len, + unsigned char *bin_data,int bin_len); + +// encode from binary to base64 ascii +// return the amount of ascii data produced +int fn_base64_bin2ascii(const unsigned char *bin_data,int bin_len, + unsigned char *ascii_data,int ascii_len); + +// URL-compatible encoding. It can be fed to standard BASE64 decoder +// after replacing: '.'->'+', '-'->'/', '_'->'=', and appending '\n' +// return the amount of ascii data produced +int fn_base64URL_bin2ascii(const unsigned char *bin_data,int bin_len, + unsigned char *ascii_data,int ascii_len); + + +#endif /*__H_FN_BASE64__*/ + + diff --git a/fn-endian.h b/fn-endian.h new file mode 100644 index 0000000..075782c --- /dev/null +++ b/fn-endian.h @@ -0,0 +1,35 @@ +/* + * frontier client endian header + * + * Author: Dave Dykstra + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ +#ifndef __HEADER_H_FN_ENDIAN_H +#define __HEADER_H_FN_ENDIAN_H + +/* __BIG_ENDIAN__ or __LITTLE_ENDIAN__ are predefined on Mac OSX + and endian.h doesn't exist there */ + +#ifdef __BIG_ENDIAN__ +#define __BIG_ENDIAN 4321 +#define __LITTLE_ENDIAN 1234 +#define __BYTE_ORDER __BIG_ENDIAN +#else +#ifdef __LITTLE_ENDIAN__ +#define __BIG_ENDIAN 4321 +#define __LITTLE_ENDIAN 1234 +#define __BYTE_ORDER __LITTLE_ENDIAN +#else +#include +#endif +#endif + +#endif /*__HEADER_H_FN_ENDIAN_H*/ diff --git a/fn-fileget.c b/fn-fileget.c new file mode 100644 index 0000000..e6d5237 --- /dev/null +++ b/fn-fileget.c @@ -0,0 +1,233 @@ +/* + * command line tool for getting files from a frontier server + * + * Author: Dave Dykstra + * + * $Id$ + * + * Copyright (c) 2011, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "frontier_client/frontier.h" + +#define RESULT_TYPE_ARRAY 6 +#define RESULT_TYPE_EOR 7 + +void usage() + { + fprintf(stderr,"Usage: fn-fileget [-c connect_string] [-r|-R] filepath ...\n"); + fprintf(stderr," -c connect_string: use given connect_string (default from environment)\n"); + fprintf(stderr," -r: request short time-to-live\n"); + fprintf(stderr," -R: request forever time-to-live\n"); + exit(2); + } + +char * +doubleurlencode(char *frombuf) + { + int c,n; + char *tobuf=malloc(strlen(frombuf)*5+1); + char *p=tobuf; + while((c=*frombuf++)!=0) + { + if(isalnum(c)||(c=='/')||(c=='-')||(c=='_')||(c=='.')) + *p++=c; + else + { + // %25 is the encoding of % + *p++='%'; + *p++='2'; + *p++='5'; + n=(c>>4); + if(n<10) + *p++=n+'0'; + else + *p++=n-10+'a'; + n=(c&0xf); + if(n<10) + *p++=n+'0'; + else + *p++=n-10+'a'; + } + } + *p='\0'; + return tobuf; + } + +int main(int argc, char **argv) + { + int ec,c,i; + unsigned long channel; + int ttl=2; + int ziplevel=0; + char zipstr[5]; + FrontierConfig *config; + char *connectstr=""; + + while(1) + { + c=getopt(argc,argv,":c:rR"); + switch(c) + { + case 'c': + connectstr=optarg; + break; + case 'r': + ttl=1; + break; + case 'R': + ttl=3; + break; + case -1: + break; + case ':': + fprintf(stderr,"Missing argument\n"); + usage(); + default: + fprintf(stderr,"Unrecognized option\n"); + usage(); + } + if(c==-1) + break; + } + + if(optind>=argc) + { + fprintf(stderr,"No files requested\n"); + usage(); + } + + /* default zip level to 0, unlike ordinary frontier client */ + if(getenv("FRONTIER_RETRIEVEZIPLEVEL")==0) + putenv("FRONTIER_RETRIEVEZIPLEVEL=0"); + + if(frontier_init(malloc,free)!=0) + { + fprintf(stderr,"Error initializing frontier client: %s\n",frontier_getErrorMsg()); + return 2; + } + ec=FRONTIER_OK; + config=frontierConfig_get(connectstr,"",&ec); + if(ec!=FRONTIER_OK) + { + fprintf(stderr,"Error getting frontierConfig object: %s\n",frontier_getErrorMsg()); + return 2; + } + channel=frontier_createChannel2(config,&ec); + if(ec!=FRONTIER_OK) + { + fprintf(stderr,"Error creating frontier channel: %s\n",frontier_getErrorMsg()); + return 2; + } + + frontier_setTimeToLive(channel,ttl); + ziplevel=frontier_getRetrieveZipLevel(channel); + if(ziplevel==0) + zipstr[0]='\0'; + else + { + zipstr[0]='z'; + zipstr[1]='i'; + zipstr[2]='p'; + zipstr[3]='0'+ziplevel; + zipstr[4]='\0'; + } + + for(i=optind;i + +#include "chashtable/hashtable.c" + +DEFINE_HASHTABLE_INSERT(fn_hashtable_insert,char,fn_hashval) +DEFINE_HASHTABLE_SEARCH(fn_hashtable_search,char,fn_hashval) + +static unsigned int +hashfromkey(void *k) + { + // This is the sdbm algorithm, generally praised on the internet as + // a good string hash + unsigned int hash=0; + unsigned char ch,*p=k; + while((ch=*p++)!=0) + hash=ch+(hash<<6)+(hash<<16)-hash; + return hash; + } + +static int +equalkeys(void *k1,void *k2) + { + return(strcmp((char *)k1,(char *)k2)==0); + } + +fn_hashtable * +fn_inithashtable() + { + return(create_hashtable(256,hashfromkey,equalkeys)); + } diff --git a/fn-hash.h b/fn-hash.h new file mode 100644 index 0000000..74133e6 --- /dev/null +++ b/fn-hash.h @@ -0,0 +1,57 @@ +/* + * frontier hash table header + * + * Author: Dave Dykstra + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ +#ifndef __HEADER_H_FN_HASH_H +#define __HEADER_H_FN_HASH_H + +/* in order to avoid symbol clashes, re-define external symbols + in this off-the-net hashtable package */ + +/* these may be used directly */ +#define hashtable_destroy fn_hashtable_destroy +#define hashtable_count fn_hashtable_count +#define hashtable_stats fn_hashtable_stats +#define hashtable s_fn_hashtable +typedef struct s_fn_hashtable fn_hashtable; + +/* don't use these directly in frontier, hint is they have double underscore */ +#define create_hashtable fn__create_hashtable +#define hashtable_insert fn__hashtable_insert +#define hashtable_search fn__hashtable_search +#define hashtable_remove fn__hashtable_remove +#define hashtable_hash fn__hashtable_hash +#define hashtable_iterator fn__hashtable_iterator +#define hashtable_iterator_key fn__hashtable_iterator_key +#define hashtable_iterator_value fn__hashtable_iterator_value +#define hashtable_iterator_advance fn__hashtable_iterator_advance +#define hashtable_iterator_remove fn__hashtable_iterator_remove +#define hashtable_iterator_search fn__hashtable_iterator_search +#define hashtable_change fn__hashtable_change + +#include "chashtable/hashtable.h" + +struct s_fn_hashval + { + int len; + char *data; + }; +typedef struct s_fn_hashval fn_hashval; + +/* the functions these define also may be used directly */ +int fn_hashtable_insert(fn_hashtable *h,char *key,fn_hashval *val); +fn_hashval *fn_hashtable_search(fn_hashtable *h,char *key); + +extern fn_hashtable *fn_inithashtable(); + +#endif /*__HEADER_H_FN_HASH_H*/ diff --git a/fn-internal.h b/fn-internal.h new file mode 100644 index 0000000..b249041 --- /dev/null +++ b/fn-internal.h @@ -0,0 +1,185 @@ +/* + * frontier client internal header + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#ifndef __HEADER_H_FN_INTERNAL_H +#define __HEADER_H_FN_INTERNAL_H + +#include + +#define FRONTIER_ID_SIZE 256 + +# define FRONTIER_SUCCESS 0 +# define FRONTIER_FAILURE 1 + +#include "fn-base64.h" +#include "openssl/md5.h" +#include "openssl/sha.h" +#include + +struct s_FrontierMemBuf + { + struct s_FrontierMemBuf *nextbuf; + size_t len; + // data follows immediately after + }; +typedef struct s_FrontierMemBuf FrontierMemBuf; +struct s_FrontierMemData + { + FrontierMemBuf *firstbuf; + FrontierMemBuf *lastbuf; + size_t total; + size_t zipped_total; + int error; + fn_b64a2b_context b64context; + unsigned char md5[MD5_DIGEST_LENGTH]; + MD5_CTX md5_ctx; + unsigned char sha256[SHA256_DIGEST_LENGTH]; + SHA256_CTX sha256_ctx; + int secured; + int binzipped; + unsigned char zipbuf[4096]; + int zipbuflen; + }; +typedef struct s_FrontierMemData FrontierMemData; +FrontierMemData *frontierMemData_create(int zipped,int secured,const char *params1,const char *params2); +int frontierMemData_finalize(FrontierMemData *md); +unsigned char *frontierMemData_getDigest(FrontierMemData *md); +void frontierMemData_delete(FrontierMemData *md); +int frontierMemData_b64append(FrontierMemData *md,const char *buf,int size); + + +struct s_FrontierPayload + { + int id; + char *encoding; + int error; + int error_code; + char *error_msg; + unsigned char *blob; + int blob_size; + unsigned int nrec; + long full_size; + char srv_md5_str[MD5_DIGEST_LENGTH*2+1]; + unsigned char digest[SHA256_DIGEST_LENGTH]; + unsigned char *srv_sig; + int srv_sig_len; + FrontierMemData *md; + }; +typedef struct s_FrontierPayload FrontierPayload; +FrontierPayload *frontierPayload_create(const char *encoding,int secured,const char *params1,const char *params2); +void frontierPayload_delete(FrontierPayload *pl); +void frontierPayload_append(FrontierPayload *pl,const char *s,int len); +int frontierPayload_finalize(FrontierPayload *pl); + +#define FNTR_WITHIN_PAYLOAD 1 + +struct s_FrontierResponse + { + int error; + int payload_num; + int error_payload_ind; + int keepalives; + int max_age; + void *parser; + int p_state; + int zipped; + int seqnum; + void *srv_rsakey; + const char *params1; + const char *params2; + FrontierPayload *payload[FRONTIER_MAX_PAYLOADNUM]; + }; +typedef struct s_FrontierResponse FrontierResponse; +FrontierResponse *frontierResponse_create(int *ec,void *srv_rsakey,const char *params1,const char *params2); +void frontierResponse_delete(FrontierResponse *fr); +int FrontierResponse_append(FrontierResponse *fr,char *buf,int len); +int frontierResponse_finalize(FrontierResponse *fr); + + +#define FRONTIER_ENV_LOG_LEVEL "FRONTIER_LOG_LEVEL" +#define FRONTIER_ENV_LOG_FILE "FRONTIER_LOG_FILE" + +struct s_fn_hashtable; +struct s_fn_client_cache_list + { + struct s_fn_client_cache_list *next; + struct s_fn_hashtable *table; + char *servlet; + }; +typedef struct s_fn_client_cache_list fn_client_cache_list; + +struct s_fn_query_stat + { + struct timespec starttime; + }; +typedef struct s_fn_query_stat fn_query_stat; + +struct s_Channel + { + FrontierConfig *cfg; + FrontierResponse *resp; + FrontierHttpClnt *ht_clnt; + pid_t pid; + int http_resp_code; + int refresh; // Refresh requested (1=soft, 2=hard) + int max_age; // Max cache age for soft refresh + int ttl; // Time-to-live requested by API (1=short, 2=long, 3=forever) + int seqnum; // sequence number for the channel + int response_seqnum; // next sequence number for responses using this channel + fn_client_cache_list *client_cache; + char *client_cache_buf; + char *ttlshort_suffix; + char *ttllong_suffix; + char *ttlforever_suffix; + int client_cache_maxsize; + void *serverrsakey[FRONTIER_MAX_SERVERN]; + fn_query_stat query_stat; + unsigned int query_bytes; + }; +typedef struct s_Channel Channel; + +struct s_RSBlob + { + FrontierResponse *resp; + int payload_error; + const char *payload_msg; + const unsigned char *buf; + unsigned int size; + unsigned int pos; + unsigned int nrec; + int respnum; + }; +typedef struct s_RSBlob RSBlob; + + + +char *frontier_str_now(char *); + +extern char *frontier_log_file; +extern int frontier_log_dup; +extern int frontier_log_level; +extern pid_t frontier_pid; +extern void *(*frontier_mem_alloc)(size_t size); +extern void (*frontier_mem_free)(void *p); +extern void (*frontier_mem_free)(void *ptr); + +void frontier_statistics_start_debug(); +void frontier_statistics_stop_debug(); +void frontier_statistics_start_query(fn_query_stat *query_stat); +void frontier_statistics_stop_query(fn_query_stat *query_stat,unsigned int response_bytes); + +#endif /*__HEADER_H_FN_INTERNAL_H*/ + + diff --git a/fn-mem.c b/fn-mem.c new file mode 100644 index 0000000..6347842 --- /dev/null +++ b/fn-mem.c @@ -0,0 +1,97 @@ +/* + * frontier client memory allocation handler + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#include +#include +#include +#include + +struct mem_chunk + { + void *ptr; + int size; + }; + +static struct mem_chunk a_mem[1024]; +static int max_mem=0; +static int mem_count=0; + +void *frontier_malloc(size_t size) + { + void *ret; + int i; + + ret=malloc(size); + ++mem_count; + + for(i=0;i<=max_mem;i++) + { + if(!a_mem[i].ptr) + { + a_mem[i].ptr=ret; + a_mem[i].size=size; + break; + } + } + + if(i>max_mem) + { + max_mem++; + a_mem[max_mem].ptr=ret; + a_mem[max_mem].size=size; + } + printf("malloc 0x%016lX, count %d, id %d, size %ld\n",(unsigned long)ret,mem_count,i,(long)size); + return ret; + } + + +void frontier_free(void *ptr) + { + int i; + int size=-1; + + --mem_count; + for(i=0;i<=max_mem;i++) + { + if(a_mem[i].ptr==ptr) + { + size=a_mem[i].size; + a_mem[i].size=-1; + a_mem[i].ptr=(long)0; + break; + } + } + + if(size==-1) + { + printf("Memory corrupted 0x%016lX, count %d, id %d\n",(unsigned long)ptr,mem_count,i); + exit(1); + } + + printf("free 0x%016lX, count %d, id %d, size %d\n",(unsigned long)ptr,mem_count,i,size); + + if(mem_count<2) + { + for(i=0;i<=max_mem;i++) + { + if(a_mem[i].ptr) + { + printf("left 0x%016lX, id %d, size %d\n",(unsigned long)a_mem[i].ptr,i,a_mem[i].size); + } + } + } + + free(ptr); + } diff --git a/fn-zlib.c b/fn-zlib.c new file mode 100644 index 0000000..ca7cda0 --- /dev/null +++ b/fn-zlib.c @@ -0,0 +1,219 @@ +/* + * frontier client zlib interface functions + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ +#include "frontier_client/frontier.h" +#include "fn-zlib.h" +#include "fn-internal.h" + +#include "zlib.h" + +static z_stream *dezstream=0; +static z_stream *inzstream=0; + +static void *fn_zalloc(void *opaque,uInt items,uInt size) + { + return frontier_mem_alloc(items*size); + } + +static void fn_zfree(void *opaque,void *address) + { + frontier_mem_free(address); + } + +static void fn_decleanup() + { + if(dezstream!=0) + { + deflateEnd(dezstream); + frontier_mem_free(dezstream); + dezstream=0; + } + } + +static void fn_incleanup() + { + if(inzstream!=0) + { + inflateEnd(inzstream); + frontier_mem_free(inzstream); + inzstream=0; + } + } + +void fn_gzip_cleanup() + { + fn_decleanup(); + fn_incleanup(); + } + +long fn_gzip_str(const char *src,long src_size,char *dest,long dest_size) + { + int ret; + + if(dezstream==0) + { + // open a stream and leave it open until channel closes + // because deflateInit does many, large allocs and thrashes + dezstream=frontier_mem_alloc(sizeof(*dezstream)); + if(dezstream==0) + return FN_ZLIB_E_NOMEM; + dezstream->zalloc=fn_zalloc; + dezstream->zfree=fn_zfree; + dezstream->opaque=0; + ret=deflateInit(dezstream,9); + if(ret!=Z_OK) + { + fn_decleanup(); + if(ret==Z_MEM_ERROR) + return FN_ZLIB_E_NOMEM; + return FN_ZLIB_E_OTHER; + } + } + else + { + // reuse existing stream + ret=deflateReset(dezstream); + if(ret!=Z_OK) + { + fn_decleanup(); + return FN_ZLIB_E_OTHER; + } + } + + dezstream->next_in=(Bytef *)src; + dezstream->avail_in=(uLongf)src_size; + dezstream->next_out=(Bytef *)dest; + dezstream->avail_out=(uLongf)dest_size; + ret=deflate(dezstream,Z_FINISH); + if(ret==Z_STREAM_END) + { + // leave stream available + return dest_size-(long)dezstream->avail_out; + } + + fn_decleanup(); + if(ret==Z_BUF_ERROR) + return FN_ZLIB_E_SMALLBUF; + return FN_ZLIB_E_OTHER; + } + +int fn_gzip_str2urlenc(const char *str,int size,char **out) + { + int zsize; + unsigned char *zbuf=0; + long zret; + int ret; + int asize; + unsigned char *abuf=0; + + if(size>MAX_STR2URL_SIZE) return FN_ZLIB_E_TOOBIG; + if(str[size-1]=='\n') + size--; // don't include trailing newline + + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"encoding request [%*s]",size,str); + + zsize=(int)(((double)size)*1.001+12); + zbuf=frontier_mem_alloc(zsize); + if(!zbuf) return FN_ZLIB_E_NOMEM; + + zret=fn_gzip_str(str,size,(char *)zbuf,zsize); + if(zret<0) {ret=zret; goto end;} + + asize=((int)zret)*2+3; + abuf=frontier_mem_alloc(asize); + if(!abuf) {ret=FN_ZLIB_E_NOMEM; goto end;} + + ret=fn_base64URL_bin2ascii(zbuf,zret,abuf,asize); + if(ret<0) goto end; + + *out=(char *)abuf; + abuf=0; + +end: + if(abuf) frontier_mem_free(abuf); + if(zbuf) frontier_mem_free(zbuf); + return ret; + } + +int fn_gunzip_init() + { + int ret=Z_OK; + + if(inzstream==0) + { + // open a stream and leave it open just like with deflate above + inzstream=frontier_mem_alloc(sizeof(*inzstream)); + if(inzstream==0) + return Z_MEM_ERROR; + inzstream->zalloc=fn_zalloc; + inzstream->zfree=fn_zfree; + inzstream->opaque=0; + inzstream->next_in=Z_NULL; + inzstream->avail_in=0; + ret=inflateInit(inzstream); + if(ret!=Z_OK) + { + fn_incleanup(); + return ret; + } + } + else + { + // reuse existing stream + ret=inflateReset(inzstream); + if(ret!=Z_OK) + { + fn_incleanup(); + return ret; + } + } + + return ret; +} + +int fn_gunzip_update(unsigned char *src,int *src_size,const unsigned char *dest,int *dest_size,int final) + { + int ret; + inzstream->next_in=(Bytef *)src; + inzstream->avail_in=(uLongf)*src_size; + inzstream->next_out=(Bytef *)dest; + inzstream->avail_out=(uLongf)*dest_size; + ret=inflate(inzstream,final?Z_FINISH:Z_SYNC_FLUSH); + *dest_size=inzstream->avail_out; + *src_size=inzstream->avail_in; + if(final) + { + if(ret==Z_STREAM_END) + { + // successful finish. leave stream available for later re-use. + return Z_OK; + } + if((ret==Z_BUF_ERROR)||(ret==Z_OK)) + { + // not quite finished, but let caller call again + return Z_BUF_ERROR; + } + // unsuccessful finish, clean up stream + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"final inflate error message: %s",inzstream->msg); + fn_incleanup(); + } + else if(ret==Z_STREAM_END) + { + // input stream is finished but caller hasn't asked to finish, that's fine + return Z_OK; + } + else if((ret!=Z_OK)&&(ret!=Z_BUF_ERROR)&&(inzstream->msg!=0)) + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"inflate error message: %s",inzstream->msg); + return ret; + } diff --git a/fn-zlib.h b/fn-zlib.h new file mode 100644 index 0000000..2b02c83 --- /dev/null +++ b/fn-zlib.h @@ -0,0 +1,31 @@ +/* + * frontier client zlib support header + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ +#ifndef __H__FN_ZLIB_H +#define __H__FN_ZLIB_H + +#define MAX_STR2URL_SIZE (1024*1024) // 1MiB + +#define FN_ZLIB_E_SMALLBUF -1 +#define FN_ZLIB_E_NOMEM -2 +#define FN_ZLIB_E_OTHER -3 +#define FN_ZLIB_E_TOOBIG -4 + +long fn_gzip_str(const char *src,long src_size,char *dest,long dest_size); +void fn_gzip_cleanup(); +int fn_gunzip_init(); +int fn_gunzip_update(unsigned char *src,int *src_size,const unsigned char *dest,int *dest_size,int final); + + +#endif //__H__FN_ZLIB_H diff --git a/fnget.py b/fnget.py new file mode 100755 index 0000000..dddc2c8 --- /dev/null +++ b/fnget.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python + +# +# Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY +# All rights reserved. +# +# For details of the Fermitools (BSD) license see Fermilab-2009.txt or +# http://fermitools.fnal.gov/about/terms.html +# + +# Simple python frontier client. +# encodes input sql query (standard encoding has to be slightly modified +# for url safety), retrieves data, and decodes results +# Author: Sinisa Veseli November 2005 +# Update: Lee Lueking added --stats-only flag for perf testing +# Update: Dave Dykstra added version number into frontier id and other mods +# +# example of usage +# ./fnget.py --url=http://lxfs6043.cern.ch:8000/Frontier3D/Frontier +# --sql="select name,version from frontier_descriptors" +# +# +import sys +import urllib2 +from xml.dom.minidom import parseString +import base64 +import zlib +import string +import curses.ascii +import time +import os.path + +frontierId = "fnget.py 1.8" + +def usage(): + progName = os.path.basename(sys.argv[0]) + print "Usage:" + print " %s --url= --sql= [--no-decode]" % progName + print " [--refresh-cache] [--retrieve-ziplevel=N] [--sign]" + print " " + +frontierUrl = None +frontierQuery = None +decodeFlag = True +refreshFlag = False +statsFlag = False +retrieveZiplevel = "zip" +signParam="" +for a in sys.argv[1:]: + arg = string.split(a, "=") + if arg[0] == "--url": + frontierUrl = arg[1] + elif arg[0] == "--sql": + frontierQuery = string.join(arg[1:], "=") + elif arg[0] == "--no-decode": + decodeFlag = False + elif arg[0] == "--refresh-cache": + refreshFlag = True + elif arg[0] == "--retrieve-ziplevel": + level = string.join(arg[1:], "=") + retrieveZiplevel="zip%s" % (level) + if level == "0": + retrieveZiplevel = "" + elif arg[0] == "--sign": + signParam="&sec=sig" + elif arg[0] == "--stats-only": + statsFlag = True + else: + print "Ignoring unrecognized argument: %s" % a + +if frontierUrl is None or frontierQuery is None: + usage() + sys.exit(1) + +if statsFlag: + pass +else: + print "Using Frontier URL: ", frontierUrl + print "Query: ", frontierQuery + print "Decode results: ", decodeFlag + print "Refresh cache: ", refreshFlag + +# encode query +encQuery = base64.binascii.b2a_base64(zlib.compress(frontierQuery,9)).replace("+", ".").replace("\n","").replace("/","-").replace("=","_") + +# frontier request +frontierRequest="%s/type=frontier_request:1:DEFAULT&encoding=BLOB%s&p1=%s%s" % (frontierUrl, retrieveZiplevel, encQuery, signParam) +if statsFlag: + pass +else: + print "\nFrontier Request:\n", frontierRequest + +# add refresh header if needed +request = urllib2.Request(frontierRequest) +if refreshFlag: + request.add_header("pragma", "no-cache") + +request.add_header("X-Frontier-Id", frontierId) + +# start and time query +queryStart = time.localtime() +if statsFlag: + pass +else: + print "\nQuery started: ", time.strftime("%m/%d/%y %H:%M:%S %Z", queryStart) + +def _under_24(): + import sys + if sys.version_info[0] < 2: return True + if sys.version_info[0] == 2: + return sys.version_info[1] < 4 + return False + +#---------------- define timeout on urllib2 socket ops -------------# +# Adapted from http://code.google.com/p/timeout-urllib2/ + +from httplib import HTTPConnection as _HC +import socket +from urllib2 import HTTPHandler as _H + +def sethttptimeout(timeout): + """Use TimeoutHTTPHandler and set the timeout value. + + Args: + timeout: the socket connection timeout value. + """ + if _under_26(): + opener = urllib2.build_opener(TimeoutHTTPHandler(timeout)) + urllib2.install_opener(opener) + else: + raise Error("This python version has timeout builtin") + +def _clear(sock): + sock.close() + return None + +def _under_26(): + import sys + if sys.version_info[0] < 2: return True + if sys.version_info[0] == 2: + return sys.version_info[1] < 6 + return False + +class Error(Exception): pass + +class HTTPConnectionTimeoutError(Error): pass + +class TimeoutHTTPConnection(_HC): + """A timeout control enabled HTTPConnection. + + Inherit httplib.HTTPConnection class and provide the socket timeout + control. + """ + _timeout = None + + def __init__(self, host, port=None, strict=None, timeout=None): + """Initialize the object. + + Args: + host: the connection host. + port: optional port. + strict: strict connection. + timeout: socket connection timeout value. + """ + _HC.__init__(self, host, port, strict) + self._timeout = timeout or TimeoutHTTPConnection._timeout + if self._timeout: self._timeout = float(self._timeout) + + def connect(self): + """Perform the socket level connection. + + A new socket object will get built everytime. If the connection + object has _timeout attribute, it will be set as the socket + timeout value. + + Raises: + HTTPConnectionTimeoutError: when timeout is hit + socket.error: when other general socket errors encountered. + """ + msg = "getaddrinfo returns an empty list" + err = socket.error + for res in socket.getaddrinfo(self.host, self.port, 0, + socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + try: + try: + self.sock = socket.socket(af, socktype, proto) + if self._timeout: self.sock.settimeout(self._timeout) + if self.debuglevel > 0: + print "connect: (%s, %s)" % (self.host, self.port) + self.sock.connect(sa) + except socket.timeout, msg: + err = socket.timeout + if self.debuglevel > 0: + print 'connect timeout:', (self.host, self.port) + self.sock = _clear(self.sock) + continue + break + except socket.error, msg: + if self.debuglevel > 0: + print 'general connect fail:', (self.host, self.port) + self.sock = _clear(self.sock) + continue + break + if not self.sock: + if err == socket.timeout: + raise HTTPConnectionTimeoutError, msg + raise err, msg + +class TimeoutHTTPHandler(_H): + """A timeout enabled HTTPHandler for urllib2.""" + def __init__(self, timeout=None, debuglevel=0): + """Initialize the object. + + Args: + timeout: the socket connect timeout value. + debuglevel: the debuglevel level. + """ + _H.__init__(self, debuglevel) + TimeoutHTTPConnection._timeout = timeout + + def http_open(self, req): + """Use TimeoutHTTPConnection to perform the http_open""" + return self.do_open(TimeoutHTTPConnection, req) + +#---------------- end timeout on socket ops ----------------# + +t1 = time.time() +if _under_24(): + print >> sys.stderr, "*WARNING:* no timeout available in python older than 2.4" + result = urllib2.urlopen(request).read() +elif _under_26(): + sethttptimeout(10) + result = urllib2.urlopen(request).read() +else: + result = urllib2.urlopen(request,None,10).read() +t2 = time.time() +queryEnd = time.localtime() +if statsFlag: + duration=(t2-t1) + size=len(result) + print duration,size,size/duration +else: + print "Query ended: ", time.strftime("%m/%d/%y %H:%M:%S %Z", queryEnd) + print "Query time: %s [seconds]\n" % (t2-t1) + +if decodeFlag: + print "Query result:\n", result + dom = parseString(result) + dataList = dom.getElementsByTagName("data") + keepalives = 0 + # Control characters represent records, but I won't bother with that now, + # and will simply replace those by space. + for data in dataList: + for node in data.childNodes: + # elements may be present, combined with whitespace text + if node.nodeName == "keepalive": + # this is of type Element + keepalives += 1 + continue + # else assume of type Text + if node.data.strip() == "": + continue + if keepalives > 0: + print keepalives, "keepalives received\n" + keepalives = 0 + + row = base64.decodestring(node.data) + if retrieveZiplevel != "": + row = zlib.decompress(row) + for c in [ '\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x08', '\x09', '\x0a', '\x0b', '\x0c', '\x0d', '\x1b', '\x17' ]: + row = row.replace(c, ' ') + + print "\nFields: " + endFirstRow = row.find('\x07') + firstRow = row[:endFirstRow] + for c in firstRow: + if curses.ascii.isctrl(c): + firstRow = firstRow.replace(c, '\n') + print firstRow + + print "\nRecords:" + pos = endFirstRow + 1 + while True: + newrow = row[pos:] + endRow = newrow.find('\x07') + if endRow < 0: + break + fixedRow = newrow[:endRow] + pos = pos + endRow + 1 + fixedRow = fixedRow.replace('\n', '') + print fixedRow diff --git a/fnget2.py b/fnget2.py new file mode 100755 index 0000000..d2b3860 --- /dev/null +++ b/fnget2.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python + +# +# Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY +# All rights reserved. +# +# For details of the Fermitools (BSD) license see Fermilab-2009.txt or +# http://fermitools.fnal.gov/about/terms.html +# + +# Simple python frontier client that can do multiple queries per connection. +# encodes input sql query (standard encoding has to be slightly modified +# for url safety), retrieves data, and decodes results +# Author: Sinisa Veseli November 2005 +# Update: Lee Lueking added --stats-only flag for perf testing +# Update: Dave Dykstra modified to use HTTP/1.1 and multiple queries +# Update: Dave Dykstra modified to also support https +# +# +import sys +import httplib +from xml.dom.minidom import parseString +import base64 +import zlib +import string +import curses.ascii +import time +import os + +frontierId = "fnget2.py 1.2" + +if sys.version < '2.4': + print 'fnget2: python version must be at least 2.4!' + sys.exit(1) + +def usage(): + progName = os.path.basename(sys.argv[0]) + print "Usage:" + print " %s --server= --path= \\" % progName + print " [--no-decode] [--refresh-cache] [--retrieve-ziplevel=N] [--stats-only] \\" + print " --sql= [--sql= ...] " + print "Example:" + print " %s --server=cmsfrontier.cern.ch:8000 --path=/FrontierInt/Frontier \\" % progName + print " --sql='select 1 from dual' --sql='select 2 from dual'" + print "Example using a proxy:" + print " %s --server=cmsfrontier1.fnal.gov:3128 \\" % progName + print " --path=http://cmsfrontier.cern.ch:8000/FrontierInt/Frontier \\" + print " --sql='select 1 from dual' --sql='select 2 from dual'" + print "Example using https:" + print " %s --server=https://localhost:8443 --path=/FrontierInt/Frontier \\" % progName + print " --sql='select 1 from dual' --sql='select 2 from dual'" + print " " + +# this was copied from dbsHttpService.py +def getKeyCert(): + """ + Gets the User key and cert if they exist otherwise throws an exception + """ + + # First presendence to HOST Certificate, RARE + if os.environ.has_key('X509_HOST_CERT'): + cert = os.environ['X509_HOST_CERT'] + key = os.environ['X509_HOST_KEY'] + + # Second preference to User Proxy, very common + elif os.environ.has_key('X509_USER_PROXY'): + cert = os.environ['X509_USER_PROXY'] + key = cert + + # Third preference to User Cert/key combination + elif os.environ.has_key('X509_USER_CERT'): + cert = os.environ['X509_USER_CERT'] + key = os.environ['X509_USER_KEY'] + + # Worst case, look for proxy at default location /tmp/x509up_u$uid + else : + uid = os.getuid() + cert = '/tmp/x509up_u'+str(uid) + key = cert + + #Set but not found + if not os.path.exists(cert) or not os.path.exists(key): + raise Exception("required cert/key not found at %s or %s for user '%s'" % (cert, key, os.getlogin())) + + # All looks OK, still doesn't guarantee validity etc. + return key, cert + + +frontierServer = None +frontierPath = None +frontierQueries = [] +decodeFlag = True +refreshFlag = False +statsFlag = False +retrieveZiplevel = "zip" +for a in sys.argv[1:]: + arg = string.split(a, "=") + if arg[0] == "--server": + frontierServer = arg[1] + elif arg[0] == "--path": + frontierPath = arg[1] + elif arg[0] == "--sql": + frontierQueries.append(string.join(arg[1:], "=")) + elif arg[0] == "--no-decode": + decodeFlag = False + elif arg[0] == "--refresh-cache": + refreshFlag = True + elif arg[0] == "--retrieve-ziplevel": + level = string.join(arg[1:], "=") + retrieveZiplevel="zip%s" % (level) + if level == "0": + retrieveZiplevel = "" + elif arg[0] == "--stats-only": + statsFlag = True + else: + print "Ignoring unrecognized argument: %s" % a + +if frontierServer is None or frontierPath is None or frontierQueries is []: + usage() + sys.exit(1) + +if statsFlag: + pass +else: + print "Using Frontier http server: ", frontierServer + print "Using Frontier server path: ", frontierPath + print "Decode results: ", decodeFlag + print "Refresh cache: ", refreshFlag + +#open the http or https server connection +if frontierServer[:8] == "https://": + key, cert = getKeyCert() + connection = httplib.HTTPSConnection(frontierServer[8:], None, key, cert) +elif frontierServer[:7] == "http://": + connection = httplib.HTTPConnection(frontierServer[7:]) +else: + connection = httplib.HTTPConnection(frontierServer) + +headers = {"X-Frontier-Id": frontierId} +# add refresh header if needed +if refreshFlag: + headers["Pragma"] = "no-cache" + + +for frontierQuery in frontierQueries: + print "\n-----\nQuery: ", frontierQuery + # encode query + encQuery = base64.binascii.b2a_base64(zlib.compress(frontierQuery,9)).replace("+", ".").replace("\n","").replace("/","-").replace("=","_") + + # frontier request + frontierRequest="%s?type=frontier_request:1:DEFAULT&encoding=BLOB%s&p1=%s" % (frontierPath, retrieveZiplevel, encQuery) + if statsFlag: + pass + else: + print "\nFrontier Request:\n%s" % frontierRequest + + # start and time query + queryStart = time.localtime() + if statsFlag: + pass + else: + print "\nQuery started: ", time.strftime("%m/%d/%y %H:%M:%S %Z", queryStart) + + t1 = time.time() + connection.request("GET", frontierRequest, None, headers) + response = connection.getresponse() + + result = response.read() + t2 = time.time() + queryEnd = time.localtime() + if statsFlag: + duration=(t2-t1) + size=len(result) + print duration,size,size/duration + else: + print "Query ended: ", time.strftime("%m/%d/%y %H:%M:%S %Z", queryEnd) + print "Query time: %s [seconds]\n" % (t2-t1) + + print "Response:", response.status, response.reason + for tuple in response.getheaders(): + print " %s: %s" % (tuple[0], tuple[1]) + print + + if decodeFlag: + print "Query result:\n", result + dom = parseString(result) + dataList = dom.getElementsByTagName("data") + # Control characters represent records, but I won't bother with that now, + # and will simply replace those by space. + for data in dataList: + if data.firstChild is not None: + + row = base64.decodestring(data.firstChild.data) + if retrieveZiplevel != "": + row = zlib.decompress(row) + for c in [ '\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x08', '\x09', '\x0a', '\x0b', '\x0c', '\x0d', '\x1b', '\x17' ]: + row = row.replace(c, ' ') + + print "\nFields: " + endFirstRow = row.find('\x07') + firstRow = row[:endFirstRow] + for c in firstRow: + if curses.ascii.isctrl(c): + firstRow = firstRow.replace(c, '\n') + print firstRow + + print "\nRecords:" + pos = endFirstRow + 1 + while True: + newrow = row[pos:] + endRow = newrow.find('\x07') + if endRow < 0: + break + fixedRow = newrow[:endRow] + pos = pos + endRow + 1 + fixedRow = fixedRow.replace('\n', '') + print fixedRow + +connection.close() diff --git a/frontier-cpp.cc b/frontier-cpp.cc new file mode 100644 index 0000000..9b98ac4 --- /dev/null +++ b/frontier-cpp.cc @@ -0,0 +1,662 @@ +/* + * frontier client C++ API implementation + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#include +#include +#include +#include "frontier_client/frontier-cpp.h" +#include "frontier_client/FrontierException.hpp" +#include "frontier_client/FrontierExceptionMapper.hpp" + +extern "C" +{ +extern void *(*frontier_mem_alloc)(size_t size); +extern void (*frontier_mem_free)(void *ptr); +}; + +static std::string create_err_msg(const char *str) + { + return std::string(str)+std::string(": ")+std::string(frontier_getErrorMsg()); + } + +using namespace frontier; + + +void Request::addKey(const std::string& key,const std::string& value) + { + if(!v_key) v_key=new std::vector(); + if(!v_val) v_val=new std::vector(); + + v_key->insert(v_key->end(),key); + v_val->insert(v_val->end(),value); + } + + +Request::~Request() + { + if(v_val) {delete v_val; v_val=NULL;} + if(v_key) {delete v_key; v_key=NULL;} + } + + + +std::string Request::encodeParam(const std::string &value) + { + const char *str=value.c_str(); + char *buf; + int len; + + len=fn_gzip_str2urlenc(str,strlen(str),&buf); + if(len<0) + { + std::ostringstream oss; + oss<<"Error "<c_str(); + + channel=frontier_createChannel(server_url.c_str(),proxy_url_c,&ec); + if(ec!=FRONTIER_OK) { + FrontierExceptionMapper::throwException(ec, "Can not create frontier channel"); + } + } + +Connection::Connection(const std::list& serverUrlList, + const std::list& proxyUrlList) { + + init(); + + /* + * Create empty config struct and add server/proxies to it. + */ + int errorCode = FRONTIER_OK; + FrontierConfig* config = frontierConfig_get("", "", &errorCode); + if(errorCode != FRONTIER_OK) { + FrontierExceptionMapper::throwException(errorCode, "Can not get frontier config object"); + } + typedef std::list::const_iterator LI; + for(LI i = serverUrlList.begin(); i != serverUrlList.end(); ++i) { + errorCode = frontierConfig_addServer(config, i->c_str()); + if(errorCode != FRONTIER_OK) { + std::ostringstream oss; + oss << "Error adding frontier server " << i->c_str(); + FrontierExceptionMapper::throwException(errorCode, oss.str()); + } + } + for(LI i = proxyUrlList.begin(); i != proxyUrlList.end(); ++i) { + errorCode = frontierConfig_addProxy(config, i->c_str(), 0); + if(errorCode != FRONTIER_OK) { + std::ostringstream oss; + oss << "Error adding frontier proxy " << i->c_str(); + FrontierExceptionMapper::throwException(errorCode, oss.str()); + } + } + errorCode = FRONTIER_OK; + channel = frontier_createChannel2(config, &errorCode); + if(errorCode != FRONTIER_OK) { + FrontierExceptionMapper::throwException(errorCode, "Error creating frontier channel"); + } + +} + +Connection::~Connection() + { + frontier_closeChannel(channel); + } + +void Connection::setReload(int reload) + { + // deprecated interface + setTimeToLive(reload?1:2); + } + +void Connection::setTimeToLive(int ttl) + { + frontier_setTimeToLive(channel,ttl); + } + + +void Connection::setDefaultParams(const std::string& logicalServer, + const std::string& parameterList) + { + frontier::init(); + frontierConfig_setDefaultLogicalServer(logicalServer.c_str()); + frontierConfig_setDefaultPhysicalServers(parameterList.c_str()); + } + + +Session::Session(Connection *connection) + { + first_row=0; + uri=NULL; + internal_data=NULL; + channel=connection->channel; + have_saved_byte=0; + num_records=0; +} + +void Session::getData(const std::vector& v_req) + { + int ec; + + if(uri) {delete uri; uri=NULL;} + + if(internal_data) {frontierRSBlob_close((FrontierRSBlob*)internal_data,&ec);internal_data=NULL;} + + std::ostringstream oss; + oss<<"Frontier"; + char delim='/'; + + for(std::vector::size_type i=0;ienc) + { + case BLOB: enc="BLOB"; break; + default: throw InvalidArgument("Unknown encoding requested"); + } + oss << delim; + if(v_req[i]->is_meta) + oss << "meta="; + else + oss << "type="; + oss << v_req[i]->obj_name; + delim='&'; + oss << delim << "encoding=" << enc; + int ziplevel = frontier_getRetrieveZipLevel(channel); + if (ziplevel > 0) + oss << "zip" << ziplevel; + + if(v_req[i]->v_key) + { + for(std::vector::size_type n=0;nv_key->size();n++) + { + oss << delim << v_req[i]->v_key->operator[](n) << '=' + << v_req[i]->v_val->operator[](n); + } + } + } + + uri=new std::string(oss.str()); + //std::cout << "URL <" << *url << ">\n"; + + ec=frontier_getRawData(channel,uri->c_str()); + if(ec!=FRONTIER_OK) { + FrontierExceptionMapper::throwException(ec, "Can not get data"); + } + } + + +void Session::setCurrentLoad(int n) { + int ec=FRONTIER_OK; + first_row=0; + FrontierRSBlob *oldrsb=(static_cast(internal_data)); + FrontierRSBlob *rsb=frontierRSBlob_open(channel,oldrsb,n,&ec); + if(ec!=FRONTIER_OK) { + FrontierExceptionMapper::throwException(ec, "Can not set current load"); + } + if(oldrsb) frontierRSBlob_close(oldrsb,&ec); + + internal_data=rsb; + if(getCurrentLoadError() != FRONTIER_OK) { + throw ProtocolError(getCurrentLoadErrorMessage()); + } + num_records=frontierRSBlob_getRecNum(rsb); +} + + +int Session::getCurrentLoadError() const + { + FrontierRSBlob *rsb=(FrontierRSBlob*)internal_data; + return frontierRSBlob_payload_error(rsb); + } + + +const char* Session::getCurrentLoadErrorMessage() const + { + FrontierRSBlob *rsb=(FrontierRSBlob*)internal_data; + return frontierRSBlob_payload_msg(rsb); + } + + +unsigned int Session::getRecNum() + { + return num_records; + } + +// Get number of records. +unsigned int Session::getNumberOfRecords() + { + return num_records; + } + + +unsigned int Session::getRSBinarySize() + { + if(!internal_data) { + // this can happen at the end, return same as getRSBinaryPos + return 0; + } + FrontierRSBlob *rs=(FrontierRSBlob*)internal_data; + return frontierRSBlob_getSize(rs); + } + + +unsigned int Session::getRSBinaryPos() + { + if(!internal_data) { + // this can happen at the end, return same as getRSBinarySize + return 0; + } + FrontierRSBlob *rs=(FrontierRSBlob*)internal_data; + return frontierRSBlob_getPos(rs); + } + + +int Session::getAnyData(AnyData* buf,int not_eor) + { + buf->isNull=0; + buf->sessionp=this; + FrontierRSBlob *rs=(FrontierRSBlob*)internal_data; + int ec=FRONTIER_OK; + BLOB_TYPE dt; + + if(not_eor && isEOR()) + throw InvalidArgument("EOR has been reached"); + + if(have_saved_byte) + { + //std::cout<<"using saved byte "<<(int)saved_byte<<'\n'; + dt=saved_byte; + have_saved_byte=0; + if(dt!=BLOB_TYPE_EOR&&!internal_data) + throw InvalidArgument("Session::getAnyData() Current load is not set"); + } + else + { + if(!internal_data) + throw InvalidArgument("Session::getAnyData() Current load is not set"); + dt=frontierRSBlob_getByte(rs,&ec); + } + + //std::cout<<"Intermediate type prefix "<<(int)dt<<'\n'; + if(ec!=FRONTIER_OK) { + throw LogicError("getAnyData() failed while getting type", ec); + } + last_field_type=dt; + + if(dt&BLOB_BIT_NULL) + { + //std::cout<<"The field is NULL\n"; + buf->isNull=1; + buf->t=dt&(~BLOB_BIT_NULL); + buf->v.str.s=0; + buf->v.str.p=NULL; + return 0; + } + //std::cout<<"Extracted type prefix "<<(int)dt<<'\n'; + + char *p; + int len; + + switch(dt) + { + case BLOB_TYPE_BYTE: buf->set(frontierRSBlob_getByte(rs,&ec)); break; + case BLOB_TYPE_INT4: buf->set(frontierRSBlob_getInt(rs,&ec)); break; + case BLOB_TYPE_INT8: buf->set(frontierRSBlob_getLong(rs,&ec)); break; + case BLOB_TYPE_FLOAT: buf->set((float)frontierRSBlob_getFloat(rs,&ec)); break; + case BLOB_TYPE_DOUBLE: buf->set(frontierRSBlob_getDouble(rs,&ec)); break; + case BLOB_TYPE_TIME: buf->set(frontierRSBlob_getLong(rs,&ec)); break; + case BLOB_TYPE_ARRAY_BYTE: + len=frontierRSBlob_getInt(rs,&ec); + if(ec!=FRONTIER_OK) + throw LogicError("can not get byte array length", ec); + if(len<0) + throw LogicError("negative byte array length", ec); + p=frontierRSBlob_getByteArray(rs,len,&ec); + if(ec==FRONTIER_OK) + { + // save the next byte and replace it with a C string terminator + // this avoids having to make another copy of the data + saved_byte=frontierRSBlob_getByte(rs,&ec); + //std::cout<<"saving byte "<<(int)saved_byte<<'\n'; + have_saved_byte=1; + p[len]=0; + buf->set(len,p); + } + break; + case BLOB_TYPE_EOR: + buf->setEOR(); + break; + default: + //std::cout<<"Unknown type prefix "<<(int)dt<<'\n'; + throw InvalidArgument("unknown type prefix"); + } + if(ec!=FRONTIER_OK) + throw LogicError("can not get AnyData value", ec); + return 0; + } + +void Session::clean() + { + if(internal_data&&isEOF()) + { + int ec=FRONTIER_OK; + /* delete as soon as possible because RSBlob may hold large buffer */ + frontierRSBlob_close((FrontierRSBlob*)internal_data,&ec); + internal_data=NULL; + if(ec!=FRONTIER_OK) + throw LogicError("can not get AnyData value", ec); + } + } + + +BLOB_TYPE Session::nextFieldType() + { + if(have_saved_byte) + { + //std::cout<<"nextFieldType using saved byte "<<(int)saved_byte<<'\n'; + return saved_byte; + } + + if(!internal_data) { + throw InvalidArgument("Session::nextFieldType() Current load is not set"); + } + FrontierRSBlob *rs=(FrontierRSBlob*)internal_data; + int ec=FRONTIER_OK; + BLOB_TYPE dt=frontierRSBlob_checkByte(rs,&ec); + if(ec!=FRONTIER_OK) { + throw LogicError("getAnyData() failed while checking type", ec); + } + + return dt; + } + + +int Session::getInt() + { + AnyData ad; + + if(getAnyData(&ad)) return -1; + return ad.getInt(); + } + + +long Session::getLong() + { + AnyData ad; + if(getAnyData(&ad)) return -1; + + if(sizeof(long)==8) return ad.getLongLong(); + return ad.getInt(); + } + + +long long Session::getLongLong() + { + AnyData ad; + + if(getAnyData(&ad)) return -1; + + return ad.getLongLong(); + } + + +double Session::getDouble() + { + AnyData ad; + + if(getAnyData(&ad)) return -1; + + return ad.getDouble(); + } + + +float Session::getFloat() + { + AnyData ad; + if(getAnyData(&ad)) return -1; + + return ad.getFloat(); + } + + +long long Session::getDate() + { + AnyData ad; + + if(getAnyData(&ad)) return -1; + + return ad.getLongLong(); + } + + +std::string* Session::getString() + { + AnyData ad; + + if(getAnyData(&ad)) return NULL; + + return ad.getString(); + } + + +std::string* Session::getBlob() + { + return getString(); + } + + + +void Session::assignString(std::string *s) + { + AnyData ad; + + if(getAnyData(&ad)) + { + *s=""; + return; + } + + ad.assignString(s); + } + + + +int Session::next() + { + AnyData ad; + + if(!first_row) + { + first_row=1; + return !(isEOF()); + } + + while(1) + { + if(isEOF()) return 0; + if(getAnyData(&ad,0)) return 0; + if(isEOF()) return 0; + if(ad.isEOR()) return 1; + } + } + + + +Session::~Session() + { + int ec; + if(internal_data) {frontierRSBlob_close((FrontierRSBlob*)internal_data,&ec);internal_data=NULL;} + if(uri) delete uri; + } + + +std::vector* Session::getRawAsArrayUChar() + { + std::string *blob=getBlob(); + int len=blob->size(); + std::vector *ret=new std::vector(len); + const char *s=blob->c_str(); + for(int i=0;ioperator[](i)=s[i]; + } + delete blob; + return ret; + } + + +std::vector* Session::getRawAsArrayInt() + { + std::string *blob=getBlob(); + if(blob->size()%4) + { + delete blob; + throw InvalidArgument("Blob size is not multiple of 4 - can not convert to int[]"); + } + int len=blob->size()/4; + std::vector *ret=new std::vector(len); + const char *s=blob->c_str(); + for(int i=0;ioperator[](i)=frontier_n2h_i32(s+i*4); + } + delete blob; + return ret; + } + + +std::vector* Session::getRawAsArrayFloat() + { + std::string *blob=getBlob(); + if(blob->size()%4) + { + delete blob; + throw InvalidArgument("Blob size is not multiple of 4 - can not convert to float[]"); + } + int len=blob->size()/4; + std::vector *ret=new std::vector(len); + const char *s=blob->c_str(); + for(int i=0;ioperator[](i)=frontier_n2h_f32(s+i*4); + } + delete blob; + return ret; + } + + +std::vector* Session::getRawAsArrayDouble() + { + std::string *blob=getBlob(); + if(blob->size()%8) + { + delete blob; + throw InvalidArgument("Blob size is not multiple of 8 - can not convert to double[]"); + } + int len=blob->size()/8; + std::vector *ret=new std::vector(len); + const char *s=blob->c_str(); + for(int i=0;ioperator[](i)=frontier_n2h_d64(s+i*8); + } + delete blob; + return ret; + } + + +std::vector* Session::getRawAsArrayLong() + { + std::string *blob=getBlob(); + if(blob->size()%4) + { + delete blob; + throw InvalidArgument("Blob size is not multiple of 4 - can not convert to int32[]"); + } + int len=blob->size()/4; + std::vector *ret=new std::vector(len); + const char *s=blob->c_str(); + for(int i=0;ioperator[](i)=frontier_n2h_i32(s+i*4); + } + delete blob; + return ret; + } + +DataSource::DataSource(const std::string& server_url,const std::string* proxy_url) + : Connection(server_url, proxy_url), Session(this) + { + } + +DataSource::DataSource(const std::list& serverUrlList, + const std::list& proxyUrlList) + : Connection(serverUrlList, proxyUrlList), Session(this) + { + } + +DataSource::~DataSource() + { + // just a placeholder; parents automatically called + } + diff --git a/frontier.c b/frontier.c new file mode 100644 index 0000000..018a2fc --- /dev/null +++ b/frontier.c @@ -0,0 +1,1185 @@ +/* + * frontier client C API implementation + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fn-internal.h" +#include "fn-hash.h" +#include "fn-zlib.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int frontier_log_level; +char *frontier_log_file; +int frontier_log_dup = 0; +pid_t frontier_pid; +void *(*frontier_mem_alloc)(size_t size); +void (*frontier_mem_free)(void *ptr); +static int initialized=0; + +static char frontier_id[FRONTIER_ID_SIZE]; +static char frontier_api_version[]=FNAPI_VERSION; + +static int chan_seqnum=0; +static void channel_delete(Channel *chn); + +static fn_client_cache_list *client_cache_list=0; + + +// our own implementation of strndup +char *frontier_str_ncopy(const char *str, size_t len) + { + char *ret; + + ret=frontier_mem_alloc(len+1); + if(!ret) return ret; + bcopy(str,ret,len); + ret[len]=0; + + return ret; + } + + +// our own implementation of strdup +char *frontier_str_copy(const char *str) + { + int len=strlen(str); + char *ret; + + ret=frontier_mem_alloc(len+1); + if(!ret) return ret; + bcopy(str,ret,len); + ret[len]=0; + + return ret; + } + +/* this returns the real owner of grid jobs */ +static char *getX509Subject() + { + char *filename=getenv("X509_USER_PROXY"); + BIO *biof=NULL; + X509 *x509cert=NULL; + char *subject=NULL; + + if(filename==NULL)return NULL; + biof=BIO_new(BIO_s_file()); + if(BIO_read_filename(biof,filename)!=0) + { + x509cert=PEM_read_bio_X509_AUX(biof,0,0,0); + if(x509cert!=NULL) + subject=X509_NAME_oneline(X509_get_subject_name(x509cert),0,0); + } + + if (biof != NULL) BIO_free(biof); + if (x509cert != NULL) X509_free(x509cert); + return subject; + } + + +// get current time as a string +// buf should be a space at least 26 bytes long, according to man ctime_r +char *frontier_str_now(char *buf) + { + time_t now=time(NULL); + char *cnow=ctime_r(&now,buf); + // eliminate trailing newline + cnow[strlen(cnow)-1]='\0'; + return cnow; + } + +/* Calculate the X-Frontier-Id value. Assumes frontier_pid is set. */ +static void set_frontier_id() + { + uid_t uid; + struct passwd *pwent; + char *appId; + char *x509Subject; + char *pwname, *pwgecos; + + uid=getuid(); + pwent=getpwuid(uid); + if(pwent==NULL) + pwname=pwgecos="pwent_failed"; + else + { + pwname=pwent->pw_name; + pwgecos=pwent->pw_gecos; + } + appId=getenv("FRONTIER_ID"); + if(appId==NULL) + appId=getenv("CMSSW_VERSION"); + if(appId==NULL) + appId="client"; + x509Subject=getX509Subject(); + + snprintf(frontier_id,FRONTIER_ID_SIZE,"%s %s %d %s(%d) %s",appId,frontier_api_version,frontier_pid,pwname,uid,(x509Subject!=NULL)?x509Subject:pwgecos); + + if(x509Subject!=NULL)OPENSSL_free(x509Subject); + + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"client id: %s",frontier_id); + } + +int frontier_init(void *(*f_mem_alloc)(size_t size),void (*f_mem_free)(void *ptr)) + { + return(frontier_initdebug(f_mem_alloc,f_mem_free, + getenv(FRONTIER_ENV_LOG_FILE),getenv(FRONTIER_ENV_LOG_LEVEL))); + } + +int frontier_initdebug(void *(*f_mem_alloc)(size_t size),void (*f_mem_free)(void *ptr), + const char *logfilename, const char *loglevel) + { + if(initialized) return FRONTIER_OK; + + if(!f_mem_alloc) {f_mem_alloc=malloc; f_mem_free=free;} + if(!f_mem_free) {f_mem_alloc=malloc; f_mem_free=free;} + + frontier_mem_alloc=f_mem_alloc; + frontier_mem_free=f_mem_free; + + frontier_pid=getpid(); + + if(!loglevel) + { + frontier_log_level=FRONTIER_LOGLEVEL_NOLOG; + frontier_log_file=(char*)0; + } + else + { + if(strcasecmp(loglevel,"warning")==0 || strcasecmp(loglevel,"info")==0) frontier_log_level=FRONTIER_LOGLEVEL_WARNING; + else if(strcasecmp(loglevel,"error")==0) frontier_log_level=FRONTIER_LOGLEVEL_ERROR; + else if(strcasecmp(loglevel,"nolog")==0) frontier_log_level=FRONTIER_LOGLEVEL_NOLOG; + else frontier_log_level=FRONTIER_LOGLEVEL_DEBUG; + + if(!logfilename) + { + frontier_log_file=(char*)0; + } + else + { + if(*logfilename=='+') + { + frontier_log_dup=1; + logfilename++; + } + frontier_log_file=frontier_str_copy(logfilename); + if(!frontier_log_init()) + { + printf("Cannot open log file %s. Log is disabled.\n",logfilename); + frontier_log_level=FRONTIER_LOGLEVEL_NOLOG; + frontier_mem_free(frontier_log_file); + frontier_log_file=(char*)0; + } + } + } + + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"starting frontier client version %s",frontier_api_version); + set_frontier_id(); + + initialized=1; + + return FRONTIER_OK; + } + + +/* note that caller is responsible for creating config but channel_create2 + or later call to channel_delete is responsible for deleting it */ +static Channel *channel_create2(FrontierConfig *config, int *ec) + { + Channel *chn; + int ret; + const char *p; + fn_client_cache_list *cache_listp; + char *servlet; + int n,s; + int longfresh=0; + + chn=frontier_mem_alloc(sizeof(Channel)); + if(!chn) + { + *ec=FRONTIER_EMEM; + FRONTIER_MSG(*ec); + if(config)frontierConfig_delete(config); + return (void*)0; + } + bzero(chn,sizeof(Channel)); + + frontier_statistics_start_debug(); + + chn->seqnum=++chan_seqnum; + chn->pid=frontier_pid; + chn->cfg=config; + if(!chn->cfg) + { + *ec=FRONTIER_EMEM; + FRONTIER_MSG(*ec); + channel_delete(chn); + return (void*)0; + } + if(!chn->cfg->server_num) + { + *ec=FRONTIER_ECFG; + frontier_setErrorMsg(__FILE__,__LINE__,"no servers configured"); + channel_delete(chn); + return (void*)0; + } + + chn->ht_clnt=frontierHttpClnt_create(ec); + if(!chn->ht_clnt||*ec) + { + channel_delete(chn); + return (void*)0; + } + + n=0; + do + { + p=frontierConfig_getServerUrl(chn->cfg); + if(!p) break; + ret=frontierHttpClnt_addServer(chn->ht_clnt,p); + if(ret) + { + *ec=ret; + channel_delete(chn); + return (void*)0; + } + n++; + }while(frontierConfig_nextServer(chn->cfg)==0); + + if(n==0) + { + *ec=FRONTIER_ECFG; + frontier_setErrorMsg(__FILE__,__LINE__,"no server configured"); + channel_delete(chn); + return (void*)0; + } + + if(frontierConfig_getBalancedServers(chn->cfg)) + frontierHttpClnt_setBalancedServers(chn->ht_clnt); + + do + { + p=frontierConfig_getProxyUrl(chn->cfg); + if(!p) break; + ret=frontierHttpClnt_addProxy(chn->ht_clnt,p); + if(ret) + { + *ec=ret; + channel_delete(chn); + return (void*)0; + } + }while(frontierConfig_nextProxy(chn->cfg)==0); + + if((n=frontierConfig_getNumBalancedProxies(chn->cfg))>0) + frontierHttpClnt_setNumBalancedProxies(chn->ht_clnt,n); + + chn->client_cache_maxsize=frontierConfig_getClientCacheMaxResultSize(chn->cfg); + if(chn->client_cache_maxsize>0) + { + chn->client_cache_buf=frontier_mem_alloc(chn->client_cache_maxsize); + if(!chn->client_cache_buf) + { + *ec=FRONTIER_EMEM; + FRONTIER_MSG(*ec); + channel_delete(chn); + return (void*)0; + } + + // get the path component of one of the servers (they're all the same) + // which is the servlet name + servlet=frontierHttpClnt_curserverpath(chn->ht_clnt); + + // locate the client cache for this servlet if it exists + for(cache_listp=client_cache_list;cache_listp!=NULL;cache_listp=cache_listp->next) + { + if(strcmp(cache_listp->servlet,servlet)==0) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"using existing %s client cache for responses less than %d bytes", + servlet,chn->client_cache_maxsize); + break; + } + } + if(!cache_listp) + { + // doesn't yet exist, create a client cache and its list entry. + // note that they are never deleted. + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"creating %s client cache for responses less than %d bytes", + servlet,chn->client_cache_maxsize); + // leave room for the servlet name after the list entry + cache_listp=(fn_client_cache_list *)frontier_mem_alloc(sizeof(*cache_listp)+strlen(servlet)+1); + if(!cache_listp) + { + *ec=FRONTIER_EMEM; + FRONTIER_MSG(*ec); + channel_delete(chn); + frontier_mem_free(cache_listp); + return (void*)0; + } + cache_listp->table=fn_inithashtable(); + if(!cache_listp->table) + { + *ec=FRONTIER_EMEM; + FRONTIER_MSG(*ec); + channel_delete(chn); + frontier_mem_free(cache_listp); + return (void*)0; + } + // tack the servlet name on the end, space was allocated above + cache_listp->servlet=((char *)cache_listp)+sizeof(*cache_listp); + strcpy(cache_listp->servlet,servlet); + cache_listp->next=client_cache_list; + client_cache_list=cache_listp; + } + chn->client_cache=cache_listp; + } + + // calculate the url suffixes for short and long time-to-live, including + // &sec and &freshkey if needed + if(frontierConfig_getSecured(chn->cfg)) + s=strlen("&sec=sig"); + else + s=0; + p=frontierConfig_getFreshkey(chn->cfg); + n=strlen(p); + if(strncmp(p,"long",4)==0) + { + longfresh=1; + p+=4; + n-=4; + } + if(n>0) + n+=strlen("&freshkey="); + chn->ttlshort_suffix=(char *)frontier_mem_alloc(s+strlen("&ttl=short")+n+1); + chn->ttllong_suffix=(char *)frontier_mem_alloc(s+(longfresh?n:0)+1); + chn->ttlforever_suffix=(char *)frontier_mem_alloc(s+strlen("&ttl=forever")+1); + if(!chn->ttlshort_suffix||!chn->ttllong_suffix||!chn->ttlforever_suffix) + { + *ec=FRONTIER_EMEM; + FRONTIER_MSG(*ec); + channel_delete(chn); + return (void*)0; + } + *chn->ttlshort_suffix='\0'; + *chn->ttllong_suffix='\0'; + *chn->ttlforever_suffix='\0'; + if(s>0) + { + strcat(chn->ttlshort_suffix,"&sec=sig"); + strcat(chn->ttllong_suffix,"&sec=sig"); + strcat(chn->ttlforever_suffix,"&sec=sig"); + } + strcat(chn->ttlshort_suffix,"&ttl=short"); + // long is the default ttl so don't need to add anything to it + strcat(chn->ttlforever_suffix,"&ttl=forever"); + if(n>0) + { + strcat(chn->ttlshort_suffix,"&freshkey="); + strcat(chn->ttlshort_suffix,p); + if(longfresh) + { + strcat(chn->ttllong_suffix,"&freshkey="); + strcat(chn->ttllong_suffix,p); + } + // freshkey never applies to the forever time-to-live + } + + frontierHttpClnt_setConnectTimeoutSecs(chn->ht_clnt, + frontierConfig_getConnectTimeoutSecs(chn->cfg)); + frontierHttpClnt_setReadTimeoutSecs(chn->ht_clnt, + frontierConfig_getReadTimeoutSecs(chn->cfg)); + frontierHttpClnt_setWriteTimeoutSecs(chn->ht_clnt, + frontierConfig_getWriteTimeoutSecs(chn->cfg)); + + chn->ttl=2; // default time-to-live is "long" + *ec=FRONTIER_OK; + return chn; + } + +static Channel *channel_create(const char *srv,const char *proxy,int *ec) + { + FrontierConfig *cfg=frontierConfig_get(srv,proxy,ec); + if(!cfg) + return (void*)0; + return channel_create2(cfg,ec); + } + +static void channel_delete(Channel *chn) + { + int i; + char nowbuf[26]; + if(!chn) return; + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"closing chan %d at %s",chn->seqnum,frontier_str_now(nowbuf)); + frontierHttpClnt_delete(chn->ht_clnt); + if(chn->resp)frontierResponse_delete(chn->resp); + frontierConfig_delete(chn->cfg); + if(chn->client_cache_buf)frontier_mem_free(chn->client_cache_buf); + if(chn->ttlshort_suffix)frontier_mem_free(chn->ttlshort_suffix); + if(chn->ttllong_suffix)frontier_mem_free(chn->ttllong_suffix); + if(chn->ttlforever_suffix)frontier_mem_free(chn->ttlforever_suffix); + for(i=0;iserverrsakey[i]) + RSA_free((RSA *)chn->serverrsakey[i]); + if(chn->seqnum==chan_seqnum) + frontier_statistics_stop_debug(); + frontier_mem_free(chn); + fn_gzip_cleanup(); + frontier_log_close(); + } + + +// This function is called by gcc when unloading the shared library +__attribute__ ((destructor)) +static void frontier_fini() + { + EVP_cleanup(); + CRYPTO_cleanup_all_ex_data(); + } + + +FrontierChannel frontier_createChannel(const char *srv,const char *proxy,int *ec) + { + Channel *chn=channel_create(srv,proxy,ec); + return (unsigned long)chn; + } + +FrontierChannel frontier_createChannel2(FrontierConfig* config, int *ec) { + Channel *chn = channel_create2(config, ec); + return (unsigned long)chn; +} + + +void frontier_closeChannel(FrontierChannel fchn) + { + channel_delete((Channel*)fchn); + } + + +// 1: short time +// 2: long time +// 3: forever +void frontier_setTimeToLive(FrontierChannel u_channel,int ttl) + { + Channel *chn=(Channel*)u_channel; + if((ttl<1)||(ttl>3)) + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"ignoring bad value of TimeToLive %d on chan %d",ttl,chn->seqnum); + else if(chn->ttl!=ttl) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"changing chan %d ttl flag to %d",chn->seqnum,ttl); + chn->ttl=ttl; + } + } + + +void frontier_setReload(FrontierChannel u_channel,int reload) + { + // deprecated interface, leave for backward compatibility + // 0=long, !0=short + frontier_setTimeToLive(u_channel,reload?1:2); + } + +static int write_data(FrontierResponse *resp,void *buf,int len) + { + int ret; + ret=FrontierResponse_append(resp,buf,len); + return ret; + } + + +static int prepare_channel(Channel *chn,int curserver,const char *params1,const char *params2) + { + int ec; + + if(chn->resp) frontierResponse_delete(chn->resp); + chn->resp=frontierResponse_create(&ec,curserver<0?0:chn->serverrsakey[curserver],params1,params2); + chn->resp->seqnum=++chn->response_seqnum; + if(!chn->resp) return ec; + + return FRONTIER_OK; + } + +static char *vcb_curservername; +static int cert_verify_callback(int ok,X509_STORE_CTX *ctx) + { + if (!ok) + frontier_setErrorMsg(__FILE__,__LINE__, "error verifying server %s cert: %s",vcb_curservername,X509_verify_cert_error_string(X509_STORE_CTX_get_error(ctx))); + return ok; + } + +static int get_cert(Channel *chn,const char *uri,int curserver) + { + int ret; + int refresh; + const char *force_reload; + char *certuri=0; + char *p,*cert=0; + int n,len; + char *servername,*commonname=0; + BIO *bio=0; + X509 *x509cert=0; + X509_STORE *x509store=0; + X509_STORE_CTX *x509storectx=0; + X509_VERIFY_PARAM *x509verifyparam=0; + X509_LOOKUP *x509lookup=0; + char *x509subject=0; + GENERAL_NAMES *subjectAltNames=0; + EVP_PKEY *pubkey=0; + RSA *rsakey=0; + + ret=prepare_channel(chn,curserver,0,0); + if(ret) return ret; + + frontierHttpClnt_setPreferIpFamily(chn->ht_clnt, + frontierConfig_getPreferIpFamily(chn->cfg)); + force_reload=frontierConfig_getForceReload(chn->cfg); + refresh=0; + if((strstr(force_reload,"long")!=0)||(strstr(force_reload,"forever")!=0)) + { + refresh=1; + if(strncmp(force_reload,"soft",4)!=0) + refresh=2; + } + frontierHttpClnt_setCacheRefreshFlag(chn->ht_clnt,refresh); + frontierHttpClnt_setUrlSuffix(chn->ht_clnt,""); + frontierHttpClnt_setFrontierId(chn->ht_clnt,frontier_id); + + ret=frontierHttpClnt_open(chn->ht_clnt); + if(ret) goto end; + + p=strstr(uri,"/type="); + if(p==0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"cannot find /type= in URI: %s",uri); + return FRONTIER_EIARG; + } + len=p-uri; +#define CERTURISTR "/type=cert_request:1&encoding=pem" + certuri=frontier_mem_alloc(len+sizeof(CERTURISTR)); + if(!certuri) {ret=FRONTIER_EMEM;FRONTIER_MSG(ret);goto end;} + strncpy(certuri,uri,len); + strcpy(certuri+len,CERTURISTR); +#undef CERTURISTR + + ret=frontierHttpClnt_get(chn->ht_clnt,certuri); + if(ret) goto end; + + servername=frontierHttpClnt_curservername(chn->ht_clnt); + if((p=strchr(servername,'['))!=0) + *p='\0'; + + len=frontierHttpClnt_getContentLength(chn->ht_clnt); + if(len<=0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"no content length from server cert request to server %s",servername); + ret=FRONTIER_ESERVER; + goto end; + } + + cert=frontier_mem_alloc(len+1); + if(!cert) {ret=FRONTIER_EMEM;FRONTIER_MSG(ret);goto end;} + + n=0; + while(nht_clnt,cert+n,len-n); + if(ret<=0) + { + if(ret==0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"server cert from %s too short: expected %d bytes, got %d",servername,len,n); + ret=FRONTIER_ESERVER; + } + goto end; + } + n+=ret; + } + cert[len]='\0'; + + if(strncmp(cert,"-----BEGIN ",11)!=0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"server cert from %s bad response",servername); + ret=FRONTIER_ESERVER; + goto end; + } + + bio=BIO_new_mem_buf(cert,len); + if(!bio) {ret=FRONTIER_EMEM;FRONTIER_MSG(ret);goto end;} + x509cert=PEM_read_bio_X509(bio,0,0,0); + if(!x509cert) + { + frontier_setErrorMsg(__FILE__,__LINE__,"error reading x509 certificate from server %s",servername); + ret=FRONTIER_ESERVER; + goto end; + } + + OpenSSL_add_all_algorithms(); + x509store=X509_STORE_new(); + x509verifyparam=X509_VERIFY_PARAM_new(); + X509_VERIFY_PARAM_set_flags(x509verifyparam,X509_V_FLAG_CRL_CHECK|X509_V_FLAG_CRL_CHECK_ALL); + X509_STORE_set1_param(x509store,x509verifyparam); + X509_STORE_set_verify_cb_func(x509store,cert_verify_callback); + x509lookup=X509_STORE_add_lookup(x509store,X509_LOOKUP_hash_dir()); + if(!x509lookup) {ret=FRONTIER_EMEM;FRONTIER_MSG(ret);goto end;} + if(!X509_LOOKUP_add_dir(x509lookup,frontierConfig_getCAPath(chn->cfg),X509_FILETYPE_PEM)) + { + frontier_setErrorMsg(__FILE__,__LINE__,"error adding %s as x509 lookup dir",frontierConfig_getCAPath(chn->cfg)); + ret=FRONTIER_ECFG; + goto end; + } + + x509storectx=X509_STORE_CTX_new(); + if(!x509storectx) {ret=FRONTIER_EMEM;FRONTIER_MSG(ret);goto end;} + if(!X509_STORE_CTX_init(x509storectx,x509store,x509cert,0)) {ret=FRONTIER_EMEM;FRONTIER_MSG(ret);goto end;} + x509subject=X509_NAME_oneline(X509_get_subject_name(x509cert),0,0); + if(!x509subject) {ret=FRONTIER_EMEM;FRONTIER_MSG(ret);goto end;} + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"verifying server certificate %s",x509subject); + + // Have to put server name in a static variable for the error, unfortunately, + // because there's no way to pass a value to the callback. + vcb_curservername=servername; + if(!X509_verify_cert(x509storectx)) + { + // error message is set inside callback + ret=FRONTIER_ESERVER; + goto end; + } + + if((p=strstr(x509subject,"/CN="))!=0) + { + commonname=p+4; + if((p=strchr(commonname,'/'))!=0) + *p='\0'; + } + if((commonname!=0)&&(strcmp(commonname,servername)==0)) + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"servername %s matched CN",servername); + else + { + int numalts,i; + subjectAltNames=(GENERAL_NAMES *)X509_get_ext_d2i(x509cert,NID_subject_alt_name,0,0); + if(!subjectAltNames) + { + frontier_setErrorMsg(__FILE__,__LINE__,"server %s name does not match cert common name %s",servername,commonname?commonname:"none"); + ret=FRONTIER_ESERVER; + goto end; + } + numalts=sk_GENERAL_NAME_num(subjectAltNames); + for(i=0;itype==GEN_DNS) + { + unsigned char *dns; + ASN1_STRING_to_UTF8(&dns,altname->d.dNSName); + if(strcmp((char *)dns,servername)==0) + { + OPENSSL_free(dns); + break; + } + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"cert alternative DNS name %s didn't match server name %s",dns,servername); + OPENSSL_free(dns); + } + } + if(i==numalts) + { + frontier_setErrorMsg(__FILE__,__LINE__,"server %s name does not match cert common name %s nor any of the alternative DNS subject names",servername,commonname?commonname:"none"); + ret=FRONTIER_ESERVER; + goto end; + } + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"server name %s matched alternative DNS subject name",servername); + } + + pubkey=X509_get_pubkey(x509cert); + if(!pubkey) + { + frontier_setErrorMsg(__FILE__,__LINE__,"error extracting public key from server %s cert",servername); + ret=FRONTIER_ESERVER; + goto end; + } + + rsakey=EVP_PKEY_get1_RSA(pubkey); + if(!rsakey) + { + frontier_setErrorMsg(__FILE__,__LINE__,"error extracting rsa key from server %s cert",servername); + ret=FRONTIER_ESERVER; + goto end; + } + + ret=FRONTIER_OK; + chn->serverrsakey[curserver]=rsakey; + +end: + frontierHttpClnt_close(chn->ht_clnt); + if(certuri)frontier_mem_free(certuri); + if(x509store)X509_STORE_free(x509store); + // X509_STORE_free does X509_LOOKUP_free + if(x509storectx)X509_STORE_CTX_free(x509storectx); + if(x509verifyparam)X509_VERIFY_PARAM_free(x509verifyparam); + if(pubkey)EVP_PKEY_free(pubkey); + if(x509cert)X509_free(x509cert); + if(bio)BIO_free(bio); + if(cert)frontier_mem_free(cert); + if(x509subject)OPENSSL_free(x509subject); + if(subjectAltNames)sk_GENERAL_NAME_pop_free(subjectAltNames,GENERAL_NAME_free); + return ret; + } + +static int get_data(Channel *chn,const char *uri,const char *body,int curserver) + { + int ret=FRONTIER_OK; + char buf[8192]; + const char *force_reload; + int force_ttl; + char *url_suffix; + char *uri_copy,*data_copy; + int uri_len; + int refresh,maxage; + int client_cache_bufsize; + fn_hashval *hashval; + char statbuf[128]; + + if(!chn) + { + frontier_setErrorMsg(__FILE__,__LINE__,"wrong channel"); + return FRONTIER_EIARG; + } + + refresh=chn->refresh; + maxage=frontierConfig_getMaxAgeSecs(chn->cfg); + if(refresh) + { + // refresh requested by server in last response + if(refresh==1) + { + // soft refresh; chn->max_age contains the max cache age to request + if((maxage<0)||(maxage>chn->max_age)) + maxage=chn->max_age; + refresh=0; // because httpclient treats a refresh==1 as maxage==0 + } + // else refresh==2, fall through to do a hard refresh + } + else + { + // refresh not requested by server, see if need to force a refresh + force_reload=frontierConfig_getForceReload(chn->cfg); + if(strcmp(force_reload,"none")!=0) + { + force_ttl=0; + if(strstr(force_reload,"short")!=0) + force_ttl=1; + else if(strstr(force_reload,"long")!=0) + force_ttl=2; + else if(strstr(force_reload,"forever")!=0) + force_ttl=3; + if(force_ttl&&(chn->ttl<=force_ttl)) + { + refresh=1; + if(strncmp(force_reload,"soft",4)!=0) + { + // hard refresh if force_reload doesn't start with "soft" + refresh=2; + } + } + } + } + frontierHttpClnt_setCacheRefreshFlag(chn->ht_clnt,refresh); + frontierHttpClnt_setCacheMaxAgeSecs(chn->ht_clnt,maxage); + frontierHttpClnt_setPreferIpFamily(chn->ht_clnt, + frontierConfig_getPreferIpFamily(chn->cfg)); + if(chn->ttl==1) + url_suffix=chn->ttlshort_suffix; + else if(chn->ttl==3) + url_suffix=chn->ttlforever_suffix; + else + url_suffix=chn->ttllong_suffix; + frontierHttpClnt_setUrlSuffix(chn->ht_clnt,url_suffix); + frontierHttpClnt_setFrontierId(chn->ht_clnt,frontier_id); + + ret=prepare_channel(chn,curserver,uri,url_suffix); + if(ret) return ret; + + ret=frontierHttpClnt_open(chn->ht_clnt); + if(ret) goto end; + + if(body) ret=frontierHttpClnt_post(chn->ht_clnt,uri,body); + else ret=frontierHttpClnt_get(chn->ht_clnt,uri); + if(ret) goto end; + + if(body)client_cache_bufsize=-1; + else client_cache_bufsize=0; + + chn->query_bytes=0; + while(1) + { + ret=frontierHttpClnt_read(chn->ht_clnt,buf,8192); + if(ret<0) goto end; + if(ret==0) break; +#if 0 + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"read %d bytes from server",ret); +#endif + chn->query_bytes+=ret; + if((chn->client_cache_maxsize>0)&&(client_cache_bufsize>=0)) + { + if(ret+client_cache_bufsize<=chn->client_cache_maxsize) + { + bcopy(buf,chn->client_cache_buf+client_cache_bufsize,ret); + client_cache_bufsize+=ret; + } + else + { + //too big, don't append any more chunks that might fit + client_cache_bufsize=-1; + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"result too large for client cache"); + } + } + ret=write_data(chn->resp,buf,ret); + if(ret!=FRONTIER_OK) goto end; + } + + if(client_cache_bufsize>0) + { + uri_len=strlen(uri)+1; + if((uri_copy=frontier_mem_alloc(uri_len))==0) + { + ret=FRONTIER_EMEM; + FRONTIER_MSG(ret); + } + else if((data_copy=frontier_mem_alloc(client_cache_bufsize))==0) + { + ret=FRONTIER_EMEM; + FRONTIER_MSG(ret); + frontier_mem_free(uri_copy); + } + else if((hashval=(fn_hashval *)frontier_mem_alloc(sizeof(fn_hashval)))==0) + { + ret=FRONTIER_EMEM; + FRONTIER_MSG(ret); + frontier_mem_free(uri_copy); + frontier_mem_free(data_copy); + } + else + { + bcopy(uri,uri_copy,uri_len); + bcopy(chn->client_cache_buf,data_copy,client_cache_bufsize); + hashval->len=client_cache_bufsize; + hashval->data=data_copy; + if(!fn_hashtable_insert(chn->client_cache->table,uri_copy,hashval)) + { + frontier_setErrorMsg(__FILE__,__LINE__,"error inserting result in client cache"); + ret=FRONTIER_EMEM; + frontier_mem_free(uri_copy); + frontier_mem_free(data_copy); + frontier_mem_free(hashval); + } + else if(frontier_log_level==FRONTIER_LOGLEVEL_DEBUG) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"inserted %d byte result in %s client cache with %d byte key", + client_cache_bufsize,chn->client_cache->servlet,uri_len); + fn_hashtable_stats(chn->client_cache->table,statbuf,sizeof(statbuf)); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"%s client cache hash stats: %s",chn->client_cache->servlet,statbuf); + } + } + } + +end: + frontierHttpClnt_close(chn->ht_clnt); + + return ret; + } + + +#define ERR_LAST_BUF_SIZE 1024 + +int frontier_getRawData(FrontierChannel u_channel,const char *uri) + { + int ret; + + ret=frontier_postRawData(u_channel,uri,NULL); + + return ret; + } + + +int frontier_postRawData(FrontierChannel u_channel,const char *uri,const char *body) + { + Channel *chn=(Channel*)u_channel; + int ret=FRONTIER_OK; + fn_hashval *hashval; + FrontierHttpClnt *clnt; + char err_last_buf[ERR_LAST_BUF_SIZE]; + int curproxy,curserver; + int have_reset_serverlist=0; + pid_t pid; + char nowbuf[26]; + + if((pid=getpid())!=frontier_pid) + { + pid_t oldpid; + // process must have forked + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"process id changed to %d",(int)pid); + oldpid=frontier_pid; + frontier_pid=pid; + // switch to new log if it includes %P in the name + frontier_log_close(); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"process id changed from %d",(int)oldpid); + // re-set id to use new pid + set_frontier_id(); + } + if(pid!=chn->pid) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"dropping any chan %d persisted connection because process id changed",chn->seqnum); + chn->pid=pid; + // drop the socket because it is shared between parent and child + frontierHttpClnt_drop(chn->ht_clnt); + } + + if(!chn) + { + frontier_setErrorMsg(__FILE__,__LINE__,"wrong channel"); + return FRONTIER_EIARG; + } + + if(!body&&(chn->client_cache_maxsize>0)) + { + if((hashval=fn_hashtable_search(chn->client_cache->table,(char *)uri))!=0) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"HIT in %s client cache, skipping contacting server",chn->client_cache->servlet); + ret=prepare_channel(chn,-1,0,0); + if(ret) return ret; + return write_data(chn->resp,hashval->data,hashval->len); + } + } + + clnt=chn->ht_clnt; + + frontierHttpClnt_resetwhenold(clnt); + curproxy=frontierHttpClnt_shuffleproxygroup(clnt); + curserver=frontierHttpClnt_shuffleservergroup(clnt); + chn->refresh=0; + bzero(err_last_buf,ERR_LAST_BUF_SIZE); + + while(1) + { + // because this is a retry loop, transfer errors aren't necessarily errors + frontier_turnErrorsIntoDebugs(1); + + ret=FRONTIER_OK; + if(frontierConfig_getSecured(chn->cfg)&&(chn->serverrsakey[curserver]==0)) + { + // don't yet have the server certificate, get that first + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"getting certificate on chan %d at %s",chn->seqnum,frontier_str_now(nowbuf)); + ret=get_cert(chn,uri,curserver); + } + if(ret==FRONTIER_OK) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"querying on chan %d at %s",chn->seqnum,frontier_str_now(nowbuf)); + frontier_statistics_start_query(&chn->query_stat); + ret=get_data(chn,uri,body,curserver); + if(ret==FRONTIER_OK) + { + ret=frontierResponse_finalize(chn->resp); + frontier_turnErrorsIntoDebugs(0); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"chan %d response %d finished at %s", + chn->seqnum,chn->resp->seqnum,frontier_str_now(nowbuf)); + frontier_statistics_stop_query(&chn->query_stat,chn->query_bytes); + } + else + frontier_statistics_stop_query(&chn->query_stat,0); + } + + if(ret!=FRONTIER_OK) + { + frontier_turnErrorsIntoDebugs(0); + + snprintf(err_last_buf,ERR_LAST_BUF_SIZE,"Request %d on chan %d failed at %s: %d %s",chn->resp->seqnum,chn->seqnum,frontier_str_now(nowbuf),ret,frontier_getErrorMsg()); + if((ret==FRONTIER_EMEM)||(ret==FRONTIER_EIARG)||(ret==FRONTIER_ECFG)) + { + frontier_setErrorMsg(__FILE__,__LINE__,err_last_buf); + break; + } + + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,err_last_buf); + } + + /* The retry strategy is unfortunately quite complicated. It is + documented in detail at + https://twiki.cern.ch/twiki/bin/view/Frontier/ClientRetryStrategy + */ + + if((!chn->refresh)&&((chn->resp->max_age>0)||(ret==FRONTIER_EPROTO))) + { + // try soft refresh on same proxy or server if age is old enough; + int age,max_age; + max_age=chn->resp->max_age; + if(max_age<=0) + max_age=FRONTIER_MAX_EPROTOAGE; + age=frontierHttpClnt_getCacheAgeSecs(clnt); + if(age>max_age) + { + chn->refresh=1; + chn->max_age=max_age; + if(curproxy>=0) + { + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"Trying max cache age %d on proxy %s and server %s",max_age,frontierHttpClnt_curproxyname(clnt),frontierHttpClnt_curservername(clnt)); + continue; + } + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"Trying max cache age %d on direct connect to server %s",max_age,frontierHttpClnt_curservername(clnt)); + continue; + } + } + + if(ret==FRONTIER_OK) + break; + + if((ret==FRONTIER_EPROTO)&&(chn->refresh==1)) + { + // try hard refresh from same proxy or server + // this is needed to clear errors that have a Last-Modified time + // that does not change + chn->refresh=2; + if(curproxy>=0) + { + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"Trying hard refresh on proxy %s and server %s",frontierHttpClnt_curproxyname(clnt),frontierHttpClnt_curservername(clnt)); + continue; + } + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"Trying hard refresh on direct connect to server %s",frontierHttpClnt_curservername(clnt)); + continue; + } + + chn->refresh=0; + + if((curproxy>=0)&&(ret!=FRONTIER_ESERVER)) + { + int stay_in_proxygroup=frontierHttpClnt_usinglastproxyingroup(clnt); + if(have_reset_serverlist&&stay_in_proxygroup) + { + // This is to avoid a potential situation of alternating + // between server list resets and proxy group resets + have_reset_serverlist=0; + stay_in_proxygroup=0; + } + if((ret!=FRONTIER_ECONNECT)&&stay_in_proxygroup) + { + // At the end of the proxy group, so if there's another server then + // use it back at the beginning of the current proxy group. + // We can't tell if the problem was on the server or the proxy + // so don't mark the current server as having had an error. + curserver=frontierHttpClnt_nextserver(clnt,0); + if(curserver>=0) + { + int newproxy=frontierHttpClnt_resetproxygroup(clnt); + if(newproxy!=curproxy) + { + curproxy=newproxy; + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"Trying next server %s with first proxy %s in proxy group", + frontierHttpClnt_curservername(clnt),frontierHttpClnt_curproxyname(clnt)); + continue; + } + // there's only one proxy + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"Trying next server %s with same proxy %s", + frontierHttpClnt_curservername(clnt),frontierHttpClnt_curproxyname(clnt)); + continue; + } + // no more servers so move on to the next proxy group with the + // first server + curserver=frontierHttpClnt_resetserverlist(clnt); + curproxy=frontierHttpClnt_nextproxy(clnt,0); + if(curproxy>=0) + { + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"Trying first server %s with proxy %s in next group", + frontierHttpClnt_curservername(clnt),frontierHttpClnt_curproxyname(clnt)); + continue; + } + // no more proxies, time for direct connect + } + else + { + // select another proxy regardless of group + curproxy=frontierHttpClnt_nextproxy(clnt,(ret==FRONTIER_ECONNECT)); + if(curproxy>=0) + { + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"Trying next proxy %s with same server %s", + frontierHttpClnt_curproxyname(clnt),frontierHttpClnt_curservername(clnt)); + continue; + } + // no more proxies, try direct with all servers + curserver=frontierHttpClnt_resetserverlist(clnt); + } +trydirectconnect: + if(!frontierConfig_getFailoverToServer(chn->cfg)) + { + frontier_setErrorMsg(__FILE__,__LINE__,"No more proxies. Last error was: %s",err_last_buf); + break; + } + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"Trying direct connect to server %s",frontierHttpClnt_curservername(clnt)); + continue; + } + + // advance to the next server, either because we're out of proxies + // or it was a server error + curserver=frontierHttpClnt_nextserver(clnt,1); + if(curserver>=0) + { + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"Trying next server %s",frontierHttpClnt_curservername(clnt)); + continue; + } + + // out of servers + if((curproxy>=0)&&(ret==FRONTIER_ESERVER)) + { + // even though it was a server error on the last server, there's still + // more proxies so it could have really been a proxy error; try again + // at the beginning of the server list and advance the proxy list + curserver=frontierHttpClnt_resetserverlist(clnt); + have_reset_serverlist=1; + curproxy=frontierHttpClnt_nextproxy(clnt,0); + if(curproxy>=0) + { + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"Trying again first server %s with proxy %s", + frontierHttpClnt_curservername(clnt),frontierHttpClnt_curproxyname(clnt)); + continue; + } + // no more proxies either + goto trydirectconnect; + } + + frontier_setErrorMsg(__FILE__,__LINE__,"No more servers/proxies. Last error was: %s",err_last_buf); + break; + } + + if(ret!=FRONTIER_OK) frontierHttpClnt_clear(clnt); + + return ret; + } + +int frontier_getRetrieveZipLevel(FrontierChannel u_channel) + { + Channel *chn=(Channel*)u_channel; + + return frontierConfig_getRetrieveZipLevel(chn->cfg); + } + diff --git a/frontier_config.c b/frontier_config.c new file mode 100644 index 0000000..dff0c70 --- /dev/null +++ b/frontier_config.c @@ -0,0 +1,1200 @@ +/* + * frontier client configuration handling + * + * Author: Sinisa Veseli + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pacparser.h" +#include +#include "frontier_client/frontier_config.h" +#include "frontier_client/frontier_log.h" +#include "frontier_client/frontier_error.h" +#include "frontier_client/frontier.h" + +/* default configuration variables */ +static int default_connect_timeout_secs=-1; +static int default_read_timeout_secs=-1; +static int default_write_timeout_secs=-1; +static int default_max_age_secs=-1; +static int default_prefer_ip_family=4; +static int default_secured=0; +static char *default_capath=0; +static char *default_force_reload=0; +static char *default_freshkey=0; +static int default_retrieve_zip_level=-1; +static char *default_logical_server=0; +static char *default_physical_servers=0; + +#define ENV_BUF_SIZE 1024 + +void *(*frontier_mem_alloc)(size_t size); +void (*frontier_mem_free)(void *ptr); + +int frontier_pacparser_init(void); + +static int getNumNonBackupProxies(FrontierConfig *cfg) + { + return cfg->proxy_num-cfg->num_backupproxies; + } + +FrontierConfig *frontierConfig_get(const char *server_url,const char *proxy_url,int *errorCode) + { + FrontierConfig *cfg; + int i; + char buf[ENV_BUF_SIZE]; + char *env; + + cfg=(FrontierConfig*)frontier_mem_alloc(sizeof(FrontierConfig)); + if(!cfg) + { + *errorCode=FRONTIER_EMEM; + FRONTIER_MSG(*errorCode); + return 0; + } + bzero(cfg,sizeof(FrontierConfig)); + + + // Set initial retrieve zip level first because it may be overridden + // by a complex server string next. + frontierConfig_setRetrieveZipLevel(cfg,frontierConfig_getDefaultRetrieveZipLevel()); + + // Likewise for the timeouts and forcereload + if(default_connect_timeout_secs==-1) + { + if((env=getenv(FRONTIER_ENV_CONNECTTIMEOUTSECS))==0) + { + // The default connect timeout should be quite short, to fail over to + // the next server if one of the servers is down. + default_connect_timeout_secs=5; + } + else + default_connect_timeout_secs=atoi(env); + } + frontierConfig_setConnectTimeoutSecs(cfg,default_connect_timeout_secs); + + if(default_read_timeout_secs==-1) + { + if((env=getenv(FRONTIER_ENV_READTIMEOUTSECS))==0) + { + // Server should send data at least every 5 seconds; allow for double. + default_read_timeout_secs=10; + } + else + default_read_timeout_secs=atoi(env); + } + frontierConfig_setReadTimeoutSecs(cfg,default_read_timeout_secs); + + if(default_write_timeout_secs==-1) + { + if((env=getenv(FRONTIER_ENV_WRITETIMEOUTSECS))==0) + { + // Writes shouldn't take very long unless there is much queued, + // which doesn't happen in the frontier client. + default_write_timeout_secs=5; + } + else + default_write_timeout_secs=atoi(env); + } + frontierConfig_setWriteTimeoutSecs(cfg,default_write_timeout_secs); + + if(default_capath==0) + { + if((env=getenv("X509_CERT_DIR"))==0) + default_capath="/etc/grid-security/certificates"; + else + default_capath=env; + } + frontierConfig_setCAPath(cfg,default_capath); + + // No env variable for these + frontierConfig_setMaxAgeSecs(cfg,default_max_age_secs); + frontierConfig_setPreferIpFamily(cfg,default_prefer_ip_family); + frontierConfig_setSecured(cfg,default_secured); + + if(default_force_reload==0) + { + if((env=getenv(FRONTIER_ENV_FORCERELOAD))==0) + default_force_reload="none"; + else + default_force_reload=env; + } + frontierConfig_setForceReload(cfg,default_force_reload); + + if(default_freshkey==0) + { + if((env=getenv(FRONTIER_ENV_FRESHKEY))==0) + default_freshkey=""; + else + default_freshkey=env; + } + frontierConfig_setFreshkey(cfg,default_freshkey); + + // Default on this is not set in environment so just set it here + frontierConfig_setClientCacheMaxResultSize(cfg,FRONTIER_DEFAULT_CLIENTCACHEMAXRESULTSIZE); + + // FailoverToServer is always true unless turned off in complex server string + frontierConfig_setFailoverToServer(cfg,1); + + // Add configured server and proxy. + *errorCode=frontierConfig_addServer(cfg,server_url); + if(*errorCode!=FRONTIER_OK)goto cleanup; + *errorCode=frontierConfig_addProxy(cfg,proxy_url,0); + if(*errorCode!=FRONTIER_OK)goto cleanup; + + // Add additional servers/proxies from env variables. + *errorCode=frontierConfig_addServer(cfg,getenv(FRONTIER_ENV_SERVER)); + if(*errorCode!=FRONTIER_OK)goto cleanup; + *errorCode=frontierConfig_addProxy(cfg,getenv(FRONTIER_ENV_PROXY),0); + if(*errorCode!=FRONTIER_OK)goto cleanup; + for(i=0;iserver_num); + if (cfg->servers_balanced) + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Servers load balanced"); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Total %d proxies",cfg->proxy_num); + if (cfg->proxies_balanced) + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"%d proxies load balanced",getNumNonBackupProxies(cfg)); + + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Retrieve zip level is %d",cfg->retrieve_zip_level); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Connect timeoutsecs is %d",cfg->connect_timeout_secs); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Read timeoutsecs is %d",cfg->read_timeout_secs); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Write timeoutsecs is %d",cfg->write_timeout_secs); + if(strcmp(cfg->force_reload,"none")!=0) + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Force reload is %s",cfg->force_reload); + if(cfg->max_age_secs>=0) + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Max age secs is %d",cfg->max_age_secs); + if(*cfg->freshkey!='\0') + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Freshkey is %s",cfg->freshkey); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Client cache max result size is %d",cfg->client_cache_max_result_size); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Failover to server is %s",cfg->failover_to_server?"yes":"no"); + + return cfg; + +cleanup: + frontierConfig_delete(cfg); + return 0; + } + + +const char *frontierConfig_getServerUrl(FrontierConfig *cfg) + { + return cfg->server[cfg->server_cur]; + } + + +const char *frontierConfig_getProxyUrl(FrontierConfig *cfg) + { + if(cfg->proxy_cur>=cfg->proxy_num)return NULL; + + return cfg->proxy[cfg->proxy_cur]; + } + + +int frontierConfig_nextServer(FrontierConfig *cfg) + { + if(cfg->server_cur+1>=cfg->server_num)return -1; + ++cfg->server_cur; + //printf("Next server %d\n",cfg->server_cur); + return 0; + } + + +int frontierConfig_nextProxy(FrontierConfig *cfg) + { + if(cfg->proxy_cur>=cfg->proxy_num)return -1; // One implicit which is always "" + ++cfg->proxy_cur; + return 0; + } + + +void frontierConfig_delete(FrontierConfig *cfg) + { + int i; + + if(!cfg)return; + + for(i=0;iserver_num;i++) + { + frontier_mem_free(cfg->server[i]); + } + for(i=0;iproxy_num;i++) + { + frontier_mem_free(cfg->proxy[i]); + } + for(i=0;iproxyconfig_num;i++) + { + frontier_mem_free(cfg->proxyconfig[i]); + } + + if(cfg->force_reload!=0)frontier_mem_free(cfg->force_reload); + + if(cfg->freshkey!=0)frontier_mem_free(cfg->freshkey); + + if(cfg->capath!=0)frontier_mem_free(cfg->capath); + + frontier_mem_free(cfg); + } + +void frontierConfig_setConnectTimeoutSecs(FrontierConfig *cfg,int secs) + { + cfg->connect_timeout_secs=secs; + } + +int frontierConfig_getConnectTimeoutSecs(FrontierConfig *cfg) + { + return cfg->connect_timeout_secs; + } + +void frontierConfig_setReadTimeoutSecs(FrontierConfig *cfg,int secs) + { + cfg->read_timeout_secs=secs; + } + +int frontierConfig_getReadTimeoutSecs(FrontierConfig *cfg) + { + return cfg->read_timeout_secs; + } + +void frontierConfig_setWriteTimeoutSecs(FrontierConfig *cfg,int secs) + { + cfg->write_timeout_secs=secs; + } + +int frontierConfig_getWriteTimeoutSecs(FrontierConfig *cfg) + { + return cfg->write_timeout_secs; + } + +void frontierConfig_setMaxAgeSecs(FrontierConfig *cfg,int secs) + { + cfg->max_age_secs=secs; + } + +int frontierConfig_getMaxAgeSecs(FrontierConfig *cfg) + { + return cfg->max_age_secs; + } + +void frontierConfig_setPreferIpFamily(FrontierConfig *cfg,int ipfamily) + { + if((ipfamily!=4)&&(ipfamily!=6)) + ipfamily=0; + cfg->prefer_ip_family=ipfamily; + } + +int frontierConfig_getPreferIpFamily(FrontierConfig *cfg) + { + return cfg->prefer_ip_family; + } + +void frontierConfig_setForceReload(FrontierConfig *cfg,char *forcereload) + { + if(cfg->force_reload!=0)frontier_mem_free(cfg->force_reload); + cfg->force_reload=frontier_str_copy(forcereload); + } + +const char *frontierConfig_getForceReload(FrontierConfig *cfg) + { + return cfg->force_reload; + } + +void frontierConfig_setFreshkey(FrontierConfig *cfg,char *freshkey) + { + if(cfg->freshkey!=0)frontier_mem_free(cfg->freshkey); + cfg->freshkey=frontier_str_copy(freshkey); + } + +const char *frontierConfig_getFreshkey(FrontierConfig *cfg) + { + return cfg->freshkey; + } + +void frontierConfig_setRetrieveZipLevel(FrontierConfig *cfg,int level) + { + if(level<0) + level=0; + if(level>9) + level=9; + cfg->retrieve_zip_level=level; + } + +int frontierConfig_getRetrieveZipLevel(FrontierConfig *cfg) + { + return cfg->retrieve_zip_level; + } + +void frontierConfig_setDefaultRetrieveZipLevel(int level) + { + if(level<0) + level=0; + if(level>9) + level=9; + default_retrieve_zip_level=level; + } + +int frontierConfig_getDefaultRetrieveZipLevel() + { + if(default_retrieve_zip_level==-1) + { + /* not yet been initialized */ + char *p; + p=getenv(FRONTIER_ENV_RETRIEVEZIPLEVEL); + if((p!=0)&&(*p!='\0')) + frontierConfig_setDefaultRetrieveZipLevel(atoi(p)); + } + if(default_retrieve_zip_level==-1) + default_retrieve_zip_level=5; + return default_retrieve_zip_level; + } + +void frontierConfig_setDefaultLogicalServer(const char *logical_server) + { + if(logical_server!=default_logical_server) + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Setting default logical server to %s",logical_server); + if(default_logical_server!=0) + { + if((logical_server!=0)&& + (strcmp(logical_server,default_logical_server)==0)) + return; + frontier_mem_free(default_logical_server); + } + if(logical_server!=0) + default_logical_server=frontier_str_copy(logical_server); + else + default_logical_server=0; + } + +char *frontierConfig_getDefaultLogicalServer() + { + if(default_logical_server==0) + { + /* try the environment */ + frontierConfig_setDefaultLogicalServer(getenv(FRONTIER_ENV_LOGICALSERVER)); + } + return default_logical_server; + } + +void frontierConfig_setDefaultPhysicalServers(const char *physical_servers) + { + if(physical_servers!=default_physical_servers) + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Setting default physical servers to %s",physical_servers); + if(default_physical_servers!=0) + { + if((physical_servers!=0)&& + (strcmp(physical_servers,default_physical_servers)==0)) + return; + frontier_mem_free(default_physical_servers); + } + if(physical_servers!=0) + default_physical_servers=frontier_str_copy(physical_servers); + else + default_physical_servers=0; + } + +char *frontierConfig_getDefaultPhysicalServers() + { + if(default_physical_servers==0) + { + /* try the environment */ + frontierConfig_setDefaultPhysicalServers(getenv(FRONTIER_ENV_PHYSICALSERVERS)); + } + return default_physical_servers; + } + +void frontierConfig_setSecured(FrontierConfig *cfg,int secured) + { + cfg->secured=secured; + } + +int frontierConfig_getSecured(FrontierConfig *cfg) + { + return cfg->secured; + } + +void frontierConfig_setCAPath(FrontierConfig *cfg,char *capath) + { + if(cfg->capath!=0)frontier_mem_free(cfg->capath); + cfg->capath=frontier_str_copy(capath); + } + +char *frontierConfig_getCAPath(FrontierConfig *cfg) + { + return cfg->capath; + } + +void frontierConfig_setClientCacheMaxResultSize(FrontierConfig *cfg,int size) + { + cfg->client_cache_max_result_size=size; + } + +int frontierConfig_getClientCacheMaxResultSize(FrontierConfig *cfg) + { + return cfg->client_cache_max_result_size; + } + +void frontierConfig_setFailoverToServer(FrontierConfig *cfg,int notno) + { + cfg->failover_to_server=notno; + } + +int frontierConfig_getFailoverToServer(FrontierConfig *cfg) + { + return cfg->failover_to_server; + } + +static int frontierConfig_parseComplexServerSpec(FrontierConfig *cfg,const char* server_spec) + { + char *str=frontier_str_copy(server_spec); + char *p=str-1; + char *keyp=0,*valp=0; + int nestlevel=0; + int ret; + + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Parsing complex server spec %s",server_spec); + + if(str==NULL) + { + FRONTIER_MSG(FRONTIER_EMEM); + return FRONTIER_EMEM; + } + + if((keyp=strstr(server_spec,"(logicalserverurl="))!=NULL) + { + // save all but this keyword as the default physical servers for + // later connections + char *nextp=strchr(keyp,')'); + if(nextp!=NULL) + { + char *cp=(char *)frontier_mem_alloc(strlen(server_spec)); + if(cp!=NULL) + { + strncpy(cp,server_spec,keyp-server_spec); + strcpy(cp+(keyp-server_spec),nextp+1); + frontierConfig_setDefaultPhysicalServers(cp); + frontier_mem_free(cp); + } + } + } + + while(*++p) + { + switch(*p) + { + case '(': + if(nestlevel==0) + { + keyp=p+1; + valp=0; + } + nestlevel++; + break; + case '=': + if(nestlevel!=1) + continue; + *p='\0'; + valp=p+1; + break; + case ')': + --nestlevel; + if(nestlevel<0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"Too many close-parens in server spec"); + ret=FRONTIER_ECFG; + goto cleanup; + } + if(nestlevel>0) + continue; + *p='\0'; + if(valp==0) + { + if(keyp==p) + /* empty parens */ + continue; + frontier_setErrorMsg(__FILE__,__LINE__,"No '=' after keyword %s",keyp); + ret=FRONTIER_ECFG; + goto cleanup; + } + ret=FRONTIER_OK; + if(strcmp(keyp,"serverurl")==0) + ret=frontierConfig_addServer(cfg,valp); + else if(strcmp(keyp,"proxyurl")==0) + ret=frontierConfig_addProxy(cfg,valp,0); + else if(strcmp(keyp,"proxyconfigurl")==0) + ret=frontierConfig_addProxyConfig(cfg,valp); + else if(strcmp(keyp,"backupproxyurl")==0) + { + /* implies failovertoserver=no */ + frontierConfig_setFailoverToServer(cfg,0); + ret=frontierConfig_addProxy(cfg,valp,1); + } + else if(strcmp(keyp,"logicalserverurl")==0) + frontierConfig_setDefaultLogicalServer(valp); + else if(strcmp(keyp,"retrieve-ziplevel")==0) + frontierConfig_setRetrieveZipLevel(cfg,atoi(valp)); + else if(strcmp(keyp,"connecttimeoutsecs")==0) + frontierConfig_setConnectTimeoutSecs(cfg,atoi(valp)); + else if(strcmp(keyp,"readtimeoutsecs")==0) + frontierConfig_setReadTimeoutSecs(cfg,atoi(valp)); + else if(strcmp(keyp,"writetimeoutsecs")==0) + frontierConfig_setWriteTimeoutSecs(cfg,atoi(valp)); + else if(strcmp(keyp,"forcereload")==0) + frontierConfig_setForceReload(cfg,valp); + else if(strcmp(keyp,"maxagesecs")==0) + frontierConfig_setMaxAgeSecs(cfg,atoi(valp)); + else if(strcmp(keyp,"preferipfamily")==0) + frontierConfig_setPreferIpFamily(cfg,atoi(valp)); + else if(strcmp(keyp,"freshkey")==0) + frontierConfig_setFreshkey(cfg,valp); + else if(strcmp(keyp,"failovertoserver")==0) + frontierConfig_setFailoverToServer(cfg,(strcmp(valp,"no")!=0)); + else if(strcmp(keyp,"loadbalance")==0) + { + if(strcmp(valp,"proxies")==0) + frontierConfig_setBalancedProxies(cfg); + else if(strcmp(valp,"servers")==0) + frontierConfig_setBalancedServers(cfg); + } + else if(strcmp(keyp,"security")==0) + frontierConfig_setSecured(cfg,(strcmp(valp,"sig")==0)); + else if(strcmp(keyp,"capath")==0) + frontierConfig_setCAPath(cfg,valp); + else if(strcmp(keyp,"clientcachemaxresultsize")==0) + frontierConfig_setClientCacheMaxResultSize(cfg,atoi(valp)); + else + { + /* else ignore unrecognized keys */ + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Unrecognized keyword %s",keyp); + } + + if(ret!=FRONTIER_OK) + goto cleanup; + break; + } + } + + if(nestlevel>0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"Unmatched parentheses"); + ret=FRONTIER_ECFG; + goto cleanup; + } + + ret=FRONTIER_OK; + +cleanup: + frontier_mem_free(str); + return ret; + } + +int frontierConfig_addServer(FrontierConfig *cfg,const char* server_url) + { + const char *logical_server; + + if((server_url!=0)&&(strchr(server_url,'(')!=NULL)) + return frontierConfig_parseComplexServerSpec(cfg,server_url); + + if(!server_url) + return FRONTIER_OK; + if(!*server_url) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Empty server url."); + return FRONTIER_OK; + } + + logical_server=frontierConfig_getDefaultLogicalServer(); + if((logical_server!=0)&&(strcmp(server_url,logical_server)==0)) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Matched logical server %s",server_url); + /* replace it with the physical servers */ + frontierConfig_addServer(cfg,frontierConfig_getDefaultPhysicalServers()); + return FRONTIER_OK; + } + + /* Ready to insert new server, make sure there's room */ + if(cfg->server_num>=FRONTIER_MAX_SERVERN) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "Reached limit of %d frontier servers",FRONTIER_MAX_SERVERN); + return FRONTIER_ECFG; + } + + /* Everything ok, insert new server. */ + cfg->server[cfg->server_num]=frontier_str_copy(server_url); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__, + "Added server <%s>",cfg->server[cfg->server_num]); + cfg->server_num++; + return FRONTIER_OK; + } + +int frontierConfig_addProxy(FrontierConfig *cfg,const char* proxy_url,int backup) + { + int n; + + if(!proxy_url) + return FRONTIER_OK; + if(!*proxy_url) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Empty proxy url."); + return FRONTIER_OK; + } + + /* Ready to insert new proxy, make sure there's room */ + if(cfg->proxy_num>=FRONTIER_MAX_PROXYN) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "Reached limit of %d frontier proxies",FRONTIER_MAX_PROXYN); + return FRONTIER_ECFG; + } + + if(backup) + { + /* insert backup proxy at the end of the list */ + n=cfg->proxy_num; + cfg->num_backupproxies++; + } + else + { + /* move any backup proxies up one to make room for this non-backup proxy */ + for(n=0;nnum_backupproxies;n++) + { + cfg->proxy[cfg->proxy_num-n]=cfg->proxy[cfg->proxy_num-n-1]; + } + n=cfg->proxy_num-cfg->num_backupproxies; + } + + /* Everything ok, insert new proxy. */ + cfg->proxy[n]=frontier_str_copy(proxy_url); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__, + "Inserted proxy %d <%s>",n,cfg->proxy[n]); + cfg->proxy_num++; + return FRONTIER_OK; + } + +int frontierConfig_addProxyConfig(FrontierConfig *cfg,const char* proxyconfig_url) + { + if(!proxyconfig_url) + return FRONTIER_OK; + if(!*proxyconfig_url) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Empty proxy config url."); + return FRONTIER_OK; + } + + /* Ready to insert new proxy config, make sure there's room */ + if(cfg->proxyconfig_num>=FRONTIER_MAX_PROXYCONFIGN) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "Reached limit of %d frontier proxyconfig urls",FRONTIER_MAX_PROXYCONFIGN); + return FRONTIER_ECFG; + } + + /* Everything ok, insert new proxyconfig url. */ + if(strcmp(proxyconfig_url,"auto")==0) + proxyconfig_url="http://wpad/wpad.dat"; + cfg->proxyconfig[cfg->proxyconfig_num]=frontier_str_copy(proxyconfig_url); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__, + "Inserted proxy config %d <%s>",cfg->proxyconfig_num, + cfg->proxyconfig[cfg->proxyconfig_num]); + cfg->proxyconfig_num++; + return FRONTIER_OK; + } + +#define MAXPACSTRINGSIZE (16*4096) +#define MAXPPERRORBUFSIZE 1024 +#define PPERRORMSGPREFIX " (pacparser message: " + +struct pp_errcontext { + char buf[MAXPPERRORBUFSIZE]; + int len; +}; +static struct pp_errcontext *pp_errorcontext; + +static int fn_pp_errorvprint(const char *fmt,va_list ap) + { + struct pp_errcontext *cx=pp_errorcontext; + int start=sizeof(PPERRORMSGPREFIX)-1+cx->len; + int left=MAXPPERRORBUFSIZE-start-2; + int ret; + char *p; + + ret=vsnprintf(&cx->buf[start],left,fmt,ap); + + // replace newlines with blank + for(p=&cx->buf[start];*p;p++) + if(*p=='\n') + *p=' '; + + if(strncmp(fmt,"WARNING: ",9)==0) + { + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__, + &cx->buf[start+9]); + return ret; + } + + if(strncmp(fmt,"DEBUG: ",7)==0) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__, + &cx->buf[start+7]); + return ret; + } + + if(ret>=left) + cx->len+=left-1; + else + cx->len+=ret; + + return(ret); + } + +static char *fn_pp_getErrorMsg() + { + struct pp_errcontext *cx=pp_errorcontext; + if(cx->len==0) + return ""; + memcpy(&cx->buf[0],PPERRORMSGPREFIX,sizeof(PPERRORMSGPREFIX)-1); + cx->buf[cx->len+sizeof(PPERRORMSGPREFIX)-1]=')'; + cx->buf[cx->len+sizeof(PPERRORMSGPREFIX)]='\0'; + return(&cx->buf[0]); + } + +int frontierConfig_doProxyConfig(FrontierConfig *cfg) + { + FrontierHttpClnt *clnt; + FrontierUrlInfo *fui=0; + int curproxyconfig,nextproxyconfig; + char *proxyconfig_url; + int n,nbytes,ret=FRONTIER_OK; + char err_last_buf[MAXPPERRORBUFSIZE]; + struct pp_errcontext err_context; + char *pacstring=0; + char *ipaddr,*proxylist; + char *p,*endp,endc; + int gotdirect=0; + int trynextloglevel; + + if(cfg->proxyconfig_num<=0) + return FRONTIER_OK; + + clnt=frontierHttpClnt_create(&ret); + if(ret!=FRONTIER_OK) + return ret; + if(!clnt) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "error creating http client object",0); + return FRONTIER_ECFG; + } + frontierHttpClnt_setPreferIpFamily(clnt, + frontierConfig_getPreferIpFamily(cfg)); + frontierHttpClnt_setConnectTimeoutSecs(clnt, + frontierConfig_getConnectTimeoutSecs(cfg)); + frontierHttpClnt_setReadTimeoutSecs(clnt, + frontierConfig_getReadTimeoutSecs(cfg)); + frontierHttpClnt_setWriteTimeoutSecs(clnt, + frontierConfig_getWriteTimeoutSecs(cfg)); + frontierHttpClnt_setFrontierId(clnt,FNAPI_VERSION); + + for(curproxyconfig=0;curproxyconfigproxyconfig_num;curproxyconfig++) + { + proxyconfig_url=cfg->proxyconfig[curproxyconfig]; + + if(strncmp(proxyconfig_url,"file://",7)==0) + { + // don't consider any proxyconfig URLs after this one + break; + } + + if(strncmp(proxyconfig_url,"http://",7)!=0) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "proxyconfigurl %s does not begin with 'http://', 'file://', or 'auto'",proxyconfig_url); + ret=FRONTIER_ECFG; + goto cleanup; + } + + // check for something beyond a server name because otherwise + // frontierHttpClnt_addServer will give wrong message about missing servlet + if(strchr(proxyconfig_url+7,'/')==0) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "config error: proxyconfigurl %s missing path on server",proxyconfig_url); + ret=FRONTIER_ECFG; + goto cleanup; + } + + ret=frontierHttpClnt_addServer(clnt,proxyconfig_url); + if(ret!=FRONTIER_OK) goto cleanup; + } + + pacstring=frontier_mem_alloc(MAXPACSTRINGSIZE+1); + if(!pacstring) + { + ret=FRONTIER_EMEM; + goto cleanup; + } + + // Messages below have either the http client servername or the current + // proxyconfig_url. In general, warning messages should have the + // servername because that includes an IP address, and warning messages + // here are about server errors. Error messages should show the + // proxyconfig_url because that's more relevant to the user. + + curproxyconfig=0; + while(1) + { + proxyconfig_url=cfg->proxyconfig[curproxyconfig]; + + if(strncmp(proxyconfig_url,"file://",7)==0) + { + char *fname; + int fd; + fname=strchr(proxyconfig_url+7,'/'); + if(fname==NULL) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "bad format for file url %s, no path", proxyconfig_url); + ret=FRONTIER_ECFG; + goto cleanup; + } + fd=open(fname,O_RDONLY); + if(fd<0) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "error opening %s: %s",fname,strerror(errno)); + ret=FRONTIER_ECFG; + goto cleanup; + } + nbytes=read(fd,pacstring,MAXPACSTRINGSIZE); + if(nbytes<0) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "error reading %s: %s",fname,strerror(errno)); + ret=FRONTIER_ECFG; + close(fd); + goto cleanup; + } + close(fd); + if(nbytes==MAXPACSTRINGSIZE) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "config error: proxyconfig file %s larger than limit of %d bytes", + fname,MAXPACSTRINGSIZE); + ret=FRONTIER_ECFG; + goto cleanup; + } + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__, + "proxyconfig file from %s is %d bytes long",fname,nbytes); + break; + } + + frontier_turnErrorsIntoDebugs(1); + trynextloglevel=FRONTIER_LOGLEVEL_WARNING; + + ret=frontierHttpClnt_open(clnt); + if(ret!=FRONTIER_OK) + { + if (strstr(frontier_getErrorMsg(),"Name or service not known")!=0) + trynextloglevel=FRONTIER_LOGLEVEL_DEBUG; + frontier_log(trynextloglevel,__FILE__,__LINE__, + "unable to connect to proxyconfig server %s: %s", + frontierHttpClnt_curservername(clnt), frontier_getErrorMsg()); + goto trynext; + } + + ret=frontierHttpClnt_get(clnt,""); + if(ret!=FRONTIER_OK) + { + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__, + "unable to get proxyconfig from %s: %s", + frontierHttpClnt_curservername(clnt), frontier_getErrorMsg()); + goto trynext; + } + + nbytes=0; + while((n=frontierHttpClnt_read(clnt,pacstring+nbytes,MAXPACSTRINGSIZE-nbytes))>0) + { + nbytes+=n; + if(nbytes==MAXPACSTRINGSIZE) + { + frontier_turnErrorsIntoDebugs(0); + frontier_setErrorMsg(__FILE__,__LINE__, + "config error: downloaded proxyconfig file from %s larger than limit of %d bytes", + proxyconfig_url,MAXPACSTRINGSIZE); + ret=FRONTIER_ECFG; + goto cleanup; + } + } + if(n<0) + { + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__, + "problem reading proxyconfig from %s: %s", + frontierHttpClnt_curservername(clnt), frontier_getErrorMsg()); + goto trynext; + } + pacstring[nbytes]='\0'; + + frontier_turnErrorsIntoDebugs(0); + + // successfully read the proxy autoconfig file + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__, + "proxyconfig file from %s is %d bytes long", + frontierHttpClnt_curservername(clnt),nbytes); + break; + +trynext: + strncpy(err_last_buf,frontier_getErrorMsg(),sizeof(err_last_buf)-1); + err_last_buf[sizeof(err_last_buf)-1]='\0'; + + frontierHttpClnt_close(clnt); + frontier_turnErrorsIntoDebugs(0); + + if((nextproxyconfig=frontierHttpClnt_nextserver(clnt,1))<0) + { + curproxyconfig++; + if ((curproxyconfigproxyconfig_num)&& + (strncmp(cfg->proxyconfig[curproxyconfig],"file://",7)==0)) + { + // there is a file:// URL still to try + frontier_log(trynextloglevel,__FILE__,__LINE__, + "Trying next proxyconfig url %s", cfg->proxyconfig[curproxyconfig]); + } + else + { + frontier_setErrorMsg(__FILE__,__LINE__, + "config error: failed to read a proxyconfig url. Last url was %s and last error was: %s", + proxyconfig_url,err_last_buf); + goto cleanup; + } + } + else + { + curproxyconfig=nextproxyconfig; + frontier_log(trynextloglevel,__FILE__,__LINE__, + "Trying next proxyconfig server %s", + frontierHttpClnt_curservername(clnt)); + } + } + + ret=frontier_pacparser_init(); + if(ret!=FRONTIER_OK) + goto cleanup; + + err_context.len=0; + err_context.buf[0]='\0'; + pp_errorcontext=&err_context; + pacparser_set_error_printer(&fn_pp_errorvprint); + + if(!pacparser_init()) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "config error: cannot initialize pacparser%s",fn_pp_getErrorMsg()); + ret=FRONTIER_ECFG; + goto cleanup; + } + + if(strncmp(proxyconfig_url,"file://",7)!=0) + { + if((ipaddr=frontierHttpClnt_myipaddr(clnt))==NULL) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "config error: error determining my IP address for proxyconfig server%s", + frontierHttpClnt_curservername(clnt),fn_pp_getErrorMsg()); + ret=FRONTIER_ECFG; + goto cleanup; + } + + pacparser_setmyip(ipaddr); + } + + if(!pacparser_parse_pac_string(pacstring)) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "config error: failure parsing %d byte proxyconfig from %s%s", + nbytes,proxyconfig_url,fn_pp_getErrorMsg()); + ret=FRONTIER_ECFG; + goto cleanup; + } + + if(cfg->server_num<1) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "config error: cannot process proxyconfigurl without a serverurl"); + ret=FRONTIER_ECFG; + goto cleanup; + } + + // need to parse the host out of the server URL, and in addition to + // other things, frontier_CreateUrlInfo happens to do that + fui=frontier_CreateUrlInfo(cfg->server[0],&ret); + if(!fui)goto cleanup; + proxylist=pacparser_find_proxy(cfg->server[0],fui->host); + + if(proxylist==NULL) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "config error: proxyconfigurl %s FindProxyForURL(\"%s\",\"%s\") returned no match%s", + proxyconfig_url,cfg->server[0],fui->host,fn_pp_getErrorMsg()); + ret=FRONTIER_ECFG; + goto cleanup; + } + + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__, + "FindProxyForURL(\"%s\",\"%s\") returned \"%s\"",cfg->server[0],fui->host,proxylist); + + if(getNumNonBackupProxies(cfg)>0) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__, + "Turning %d proxies into backup proxies so they will be after proxyconfig proxies",getNumNonBackupProxies(cfg)); + cfg->num_backupproxies=cfg->proxy_num; + } + + if(cfg->proxies_balanced) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__, + "Disabling loadbalance=proxies because of proxyconfigurl"); + cfg->proxies_balanced=0; + } + + // Now parse the returned proxy list + p=proxylist; + endc=*p; + while(endc!='\0') + { + while(*p==' ') + p++; + for(endp=p;*endp&&(*endp!=';')&&(*endp!=' ');endp++) + ; + endc=*endp; + *endp='\0'; + if(strcmp(p,"PROXY")==0) + { + if((endc=='\0')||(endc==';')) + p=endp; + else + { + p=endp+1; + while(*p==' ') + p++; + for(endp=p;*endp&&(*endp!=';')&&(*endp!=' ');endp++) + ; + endc=*endp; + *endp='\0'; + } + if(*p) + frontierConfig_addProxy(cfg,p,0); + else + { + frontier_setErrorMsg(__FILE__,__LINE__, + "config error: proxyconfigurl %s FindProxyForURL(\"%s\",\"%s\") returned \"PROXY\" without a proxy", + proxyconfig_url,cfg->server[0],fui->host); + ret=FRONTIER_ECFG; + goto cleanup; + } + } + else if(strcmp(p,"DIRECT")==0) + { + gotdirect=1; + break; + } + else if(*p) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "config error: proxyconfigurl %s FindProxyForURL(\"%s\",\"%s\") returned unrecognized type %s, expect PROXY or DIRECT", + proxyconfig_url,cfg->server[0],fui->host,p); + ret=FRONTIER_ECFG; + goto cleanup; + } + p=endp+1; + } + + if(!gotdirect) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__, + "no DIRECT found in proxyconfig list, setting failovertoserver=no"); + frontierConfig_setFailoverToServer(cfg,0); + } + + ret=FRONTIER_OK; + +cleanup: + if(pacstring) frontier_mem_free(pacstring); + pacparser_cleanup(); + frontierHttpClnt_close(clnt); + frontierHttpClnt_delete(clnt); + if(fui) frontier_DeleteUrlInfo(fui); + return ret; + } + +void frontierConfig_setBalancedProxies(FrontierConfig *cfg) + { + cfg->proxies_balanced=1; + } + +int frontierConfig_getNumBalancedProxies(FrontierConfig *cfg) + { + if(!cfg->proxies_balanced) + return 0; + return getNumNonBackupProxies(cfg); + } + +void frontierConfig_setBalancedServers(FrontierConfig *cfg) + { + cfg->servers_balanced=1; + } + +int frontierConfig_getBalancedServers(FrontierConfig *cfg) + { + return cfg->servers_balanced; + } + diff --git a/frontier_error.c b/frontier_error.c new file mode 100644 index 0000000..1d43a6a --- /dev/null +++ b/frontier_error.c @@ -0,0 +1,88 @@ +/* + * frontier client error message handling + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ +#include "frontier_client/frontier_error.h" +#include "frontier_client/frontier_log.h" +#include "fn-internal.h" +#include +#include +#include +#include +#include +#include +#include + +#define MSG_BUF_SIZE 1024 + +static char _frontier_error_msg[MSG_BUF_SIZE]; +static int errors_into_debugs=0; + +static const char *fn_errs[]= + { + "Ok", + "Invalid argument passed", /*FRONTIER_EIARG*/ + "mem_alloc failed", /*FRONTIER_EMEM*/ + "config error", /*FRONTIER_ECFG*/ + "system error", /*FRONTIER_ESYS*/ + "unknown error", /*FRONTIER_EUNKNOWN*/ + "error while communicating over network", /*FRONTIER_ENETWORK*/ + "protocol error (bad response, etc)", /*FRONTIER_EPROTO*/ + NULL + }; + +const char *frontier_get_err_desc(int err_num) + { + int i; + if(err_num>0) return "unknown"; + i=0; + while(fn_errs[i]) + { + if(i==(-err_num)) return fn_errs[i]; + ++i; + } + return "unknown_2"; + } + +void frontier_vsetErrorMsg(const char *file,int line,const char *fmt,va_list ap) + { + int ret,pos; + + bzero(_frontier_error_msg,MSG_BUF_SIZE); + + ret=snprintf(_frontier_error_msg,MSG_BUF_SIZE,"[%s:%d]: ",file,line); + pos=ret; + + ret+=vsnprintf(_frontier_error_msg+pos,MSG_BUF_SIZE-ret,fmt,ap); + + frontier_log(errors_into_debugs?FRONTIER_LOGLEVEL_DEBUG:FRONTIER_LOGLEVEL_ERROR, + file,line,"%s",_frontier_error_msg+pos); + } + +void frontier_setErrorMsg(const char *file,int line,const char *fmt,...) + { + va_list ap; + va_start(ap,fmt); + frontier_vsetErrorMsg(file,line,fmt,ap); + va_end(ap); + } + +const char *frontier_getErrorMsg() + { + return _frontier_error_msg; + } + +void frontier_turnErrorsIntoDebugs(int value) + { + errors_into_debugs=value; + } diff --git a/frontier_log.c b/frontier_log.c new file mode 100644 index 0000000..017cd30 --- /dev/null +++ b/frontier_log.c @@ -0,0 +1,112 @@ +/* + * frontier client log message handling + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ +#include "frontier_client/frontier_log.h" +#include "fn-internal.h" +#include +#include +#include +#include +#include +#include + +#define LOG_BUF_SIZE 1024*2 +#define MAX_LOG_PATH_SIZE 1024 + +static int log_fd=-1; + +static const char *log_desc[]= + { + "debug", + "warn ", + "error" + }; + + +int frontier_log_init() + { + char pathbuf[MAX_LOG_PATH_SIZE]; + char *fname=frontier_log_file; + char *p; + int idx; + if((p=strstr(fname,"%P"))!=NULL) + { + // replace the %P with the process id + idx=p-fname; + if(idx>MAX_LOG_PATH_SIZE)idx=MAX_LOG_PATH_SIZE; + strncpy(pathbuf,fname,idx); + if(idxMAX_LOG_PATH_SIZE)idx=MAX_LOG_PATH_SIZE; + strncpy(pathbuf+idx,p+2,MAX_LOG_PATH_SIZE-idx); + pathbuf[MAX_LOG_PATH_SIZE-1]='\0'; + fname=pathbuf; + } + log_fd=open(fname,O_CREAT|O_APPEND|O_WRONLY,0644); + if(log_fd>=0) + return 1; + return 0; + } + +void frontier_vlog(int level,const char *file,int line,const char *fmt,va_list ap) + { + int ret; + char log_msg[LOG_BUF_SIZE]; + + if(level2) level=2; + + ret=snprintf(log_msg,LOG_BUF_SIZE-1,"%s [%s:%d]: ",log_desc[level],file,line); + + ret+=vsnprintf(log_msg+ret,LOG_BUF_SIZE-ret-1,fmt,ap); + + if(ret>LOG_BUF_SIZE-2) + ret=LOG_BUF_SIZE-2; + + log_msg[ret]='\n'; + log_msg[ret+1]=0; + + if(!frontier_log_file||(frontier_log_dup&&(level>=FRONTIER_LOGLEVEL_WARNING))) + { + (void)write(1,log_msg,ret+1); + fsync(1); + if(!frontier_log_file) return; + } + if(log_fd<0) + { + (void)frontier_log_init(); + if(log_fd<0) return; + } + (void)write(log_fd,log_msg,ret+1); + } + +void frontier_log(int level,const char *file,int line,const char *fmt,...) + { + va_list ap; + va_start(ap,fmt); + frontier_vlog(level,file,line,fmt,ap); + va_end(ap); + } + +void frontier_log_close() + { + if(log_fd>=0) + { + close(log_fd); + log_fd=-1; + } + } + diff --git a/frontierqueries b/frontierqueries new file mode 100755 index 0000000..d09fb70 --- /dev/null +++ b/frontierqueries @@ -0,0 +1,130 @@ +#!/bin/bash + +# +# Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY +# All rights reserved. +# +# For details of the Fermitools (BSD) license see Fermilab-2009.txt or +# http://fermitools.fnal.gov/about/terms.html +# + +# Parse a log file created by setting FRONTIER_LOG_LEVEL=debug and +# FRONTIER_LOG_FILE=filename and print to stdout one query per line. +# The query result size is also printed in the form (NNN/XX%) where NNN +# is the full size in bytes and optionally /XX% is the compression ratio +# if it was compressed. The size is by default at the end of the +# line but with a -sizefirst option it is at the beginning. +# Before the query will be a hyphen (-) if the query is satisfied from +# the frontier internal cache, an asterisk (*) if it is a "short" +# time-to-live query that is sent out, a plus (+) if it is a "forever" +# time-to-live query, or a blank if it is a "long" time-to-live query. +# The -sort option sorts the output by full size. Implies -sizefirst. +# The -secsfirst option prints the elapsed seconds first. Otherwise the +# elapsed seconds always follows the size +# Written by Dave Dykstra +# $Id$ +# +SORT=cat +if [ "$1" = "-sizefirst" ]; then + PRINTORDER="size, elapsedseconds, mark, query" + shift +elif [ "$1" = "-sort" ]; then + # sort by size + PRINTORDER="size, elapsedseconds, mark, query" + SORT="sort -t( -k 2 -n" + shift +elif [ "$1" = "-secsfirst" ]; then + PRINTORDER="elapsedseconds, size, mark, query" + shift +else + PRINTORDER="mark, query, size, elapsedseconds" +fi +if [ $# != 1 ]; then + echo "Usage: $0 [-sizefirst|-sort|-secsfirst] frontier_client.log" >&2 + exit 1 +fi +awk ' +BEGIN{ + inquery=0 + query="" +} +function parsetime(mon,day,hhmmss,year) { + split(hhmmss,times,":") + if (mon == "Jan") {month="01"} + else if (mon == "Feb") {month="02"} + else if (mon == "Mar") {month="03"} + else if (mon == "Apr") {month="04"} + else if (mon == "May") {month="05"} + else if (mon == "Jun") {month="06"} + else if (mon == "Jul") {month="07"} + else if (mon == "Aug") {month="08"} + else if (mon == "Sep") {month="09"} + else if (mon == "Oct") {month="10"} + else if (mon == "Nov") {month="11"} + else if (mon == "Dec") {month="12"} + return mktime(year " " month " " day " " times[1] " " times[2] " " times[3]) +} +/querying on chan/{ + starttime=parsetime($9,$10,$11,$12) +} +/chan .* response .* finished at/{ + elapsedseconds=parsetime($10,$11,$12,$13)-starttime +} +/encoding request/{ + if (query != "") { + print '"$PRINTORDER"' + } + mark=" " + start=index($0, "t [") + query=substr($0,start+3) + end=index(query, "]") + if (end==0) { + end=length(query)+1 + inquery=1 + } + query=substr(query,1,end-1) + if (inquery==1) + next +} +(inquery==1){ + end=index($0, "]") + if (end==0) { + end=length($0)+1 + } + else { + inquery=0 + } + query=query substr($0,1,end-1) +} +/ + +#include +#include + + +int main(int argc, char **argv) + { + if(argc!=3) + { + std::cout<<"Usage: "< vrq; + vrq.insert(vrq.end(),&req); + ds.getData(vrq); + + ds.setCurrentLoad(1); + int nrec=ds.getRecNum(); + std::cout<<"Got "< +#include "../fn-internal.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define MAX_NAME_LEN 256 +#define URL_FMT_SRV "http://%127[^/]/%127s" +#define URL_FMT_PROXY "http://%s" + +#define PERSISTCONNECTION + +FrontierHttpClnt *frontierHttpClnt_create(int *ec) + { + FrontierHttpClnt *c; + + c=frontier_mem_alloc(sizeof(FrontierHttpClnt)); + if(!c) + { + *ec=FRONTIER_EMEM; + FRONTIER_MSG(*ec); + return c; + } + bzero(c,sizeof(FrontierHttpClnt)); + c->serveri.hosts=c->server; + c->proxyi.hosts=c->proxy; + c->socket=-1; + c->frontier_id=(char*)0; + c->data_pos=0; + c->data_size=0; + c->content_length=-1; + c->age=-1; + c->url_suffix=""; + c->max_age=-1; + + *ec=FRONTIER_OK; + return c; + } + +void frontierHttpClnt_setConnectTimeoutSecs(FrontierHttpClnt *c,int timeoutsecs) + { + c->connect_timeout_secs=timeoutsecs; + } + +void frontierHttpClnt_setReadTimeoutSecs(FrontierHttpClnt *c,int timeoutsecs) + { + c->read_timeout_secs=timeoutsecs; + } + +void frontierHttpClnt_setWriteTimeoutSecs(FrontierHttpClnt *c,int timeoutsecs) + { + c->write_timeout_secs=timeoutsecs; + } + +int frontierHttpClnt_addServer(FrontierHttpClnt *c,const char *url) + { + FrontierUrlInfo *fui; + int ec=FRONTIER_OK; + + fui=frontier_CreateUrlInfo(url,&ec); + if(!fui) return ec; + + if(!fui->path) + { + frontier_setErrorMsg(__FILE__,__LINE__,"config error: server %s: servlet path is missing",fui->host); + frontier_DeleteUrlInfo(fui); + return FRONTIER_ECFG; + } + + if(c->serveri.total>=(sizeof(c->server)/sizeof(c->server[0]))) + { + frontier_setErrorMsg(__FILE__,__LINE__,"config error: server list is too long"); + frontier_DeleteUrlInfo(fui); + return FRONTIER_ECFG; + } + + c->server[c->serveri.total]=fui; + c->serveri.total++; + + return FRONTIER_OK; + } + + +int frontierHttpClnt_addProxy(FrontierHttpClnt *c,const char *url) + { + FrontierUrlInfo *fui; + int ec=FRONTIER_OK; + + fui=frontier_CreateUrlInfo(url,&ec); + if(!fui) return ec; + + if(c->proxyi.total>=(sizeof(c->proxy)/sizeof(c->proxy[0]))) + { + frontier_setErrorMsg(__FILE__,__LINE__,"config error: proxy list is too long"); + frontier_DeleteUrlInfo(fui); + return FRONTIER_ECFG; + } + + c->proxy[c->proxyi.total]=fui; + c->proxyi.total++; + + return FRONTIER_OK; + } + + + +void frontierHttpClnt_setCacheRefreshFlag(FrontierHttpClnt *c,int refresh_flag) + { + c->refresh_flag=refresh_flag; + } + +void frontierHttpClnt_setCacheMaxAgeSecs(FrontierHttpClnt *c,int secs) + { + c->max_age=secs; + } + +void frontierHttpClnt_setPreferIpFamily(FrontierHttpClnt *c,int ipfamily) + { + c->prefer_ip_family=ipfamily; + } + +void frontierHttpClnt_setUrlSuffix(FrontierHttpClnt *c,char *suffix) + { + /* note that no copy is made -- caller must insure longevity of suffix */ + c->url_suffix=suffix; + } + + +void frontierHttpClnt_setFrontierId(FrontierHttpClnt *c,const char *frontier_id) + { + int i; + int len; + char *p; + unsigned char ch; + + len=strlen(frontier_id); + if(len>MAX_NAME_LEN) len=MAX_NAME_LEN; + + if(c->frontier_id) frontier_mem_free(c->frontier_id); + c->frontier_id=frontier_mem_alloc(len+1); + p=c->frontier_id; + + // eliminate any illegal characters + for(i=0;i/?=+-_{}[]()|@",ch)) + *p++=ch; + } + *p=0; + } + + + +static int http_read(FrontierHttpClnt *c) + { + //printf("\nhttp_read\n"); + //bzero(c->buf,FRONTIER_HTTP_BUF_SIZE); + c->data_pos=0; + c->data_size=frontier_read(c->socket,c->buf,FRONTIER_HTTP_BUF_SIZE,c->read_timeout_secs,c->cur_ai); + + return c->data_size; + } + + +static int read_char(FrontierHttpClnt *c,char *ch) + { + int ret; + + if(c->data_pos+1>c->data_size) + { + ret=http_read(c); + if(ret<=0) return ret; + } + *ch=c->buf[c->data_pos]; + c->data_pos++; + return 1; + } + + +static int test_char(FrontierHttpClnt *c,char *ch) + { + int ret; + + if(c->data_pos+1>c->data_size) + { + ret=http_read(c); + if(ret<=0) return ret; + } + *ch=c->buf[c->data_pos]; + return 1; + } + + +static int read_line(FrontierHttpClnt *c,char *buf,int buf_len) + { + int i; + int ret; + char ch,next_ch; + + i=0; + while(1) + { + ret=read_char(c,&ch); + if(ret<0) return ret; + if(!ret) break; + + if(ch=='\r' || ch=='\n') + { + ret=test_char(c,&next_ch); + if(ret<0) return ret; + if(!ret) break; + if(ch=='\r' && next_ch=='\n') read_char(c,&ch); // Just ignore + break; + } + buf[i]=ch; + i++; + if(i>=buf_len-1) break; + } + buf[i]=0; + return i; + } + + +void initrand(FrontierHttpClnt *c) + { + unsigned seed; + if(!RAND_bytes((unsigned char *)&seed,sizeof(seed))) + { + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"failure getting random number, using time"); + seed=(unsigned)time(0); + } + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"random seed: %u",seed); + c->proxyi.rand_seed=c->serveri.rand_seed=seed; + } + + +static int open_connection(FrontierHttpClnt *c) + { + int ret; + struct addrinfo *ai; + FrontierUrlInfo *fui_proxy,*fui_server,*fui; + in_port_t port; + unsigned *rand_seedp; + + if(c->socket!=-1) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"reusing persisted connection s=%d",c->socket); + return 0; + } + + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"no persisted connection found, opening new"); + + fui_proxy=c->proxy[c->proxyi.cur]; + fui_server=c->server[c->serveri.cur]; + + if(!fui_server) + { + frontier_setErrorMsg(__FILE__,__LINE__,"no available server"); + return FRONTIER_ECFG; + } + + if(fui_proxy) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"connecting to proxy %s",fui_proxy->host); + fui=fui_proxy; + c->using_proxy=1; + } + else + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"connecting to server %s",fui_server->host); + fui=fui_server; + c->using_proxy=0; + } + + rand_seedp=&c->serveri.rand_seed; + if(*rand_seedp==0) + initrand(c); + ret=frontier_resolv_host(fui,c->prefer_ip_family,rand_seedp); + if(ret) return ret; + ai=fui->fai->ai; + + c->socket=frontier_socket(ai->ai_addr->sa_family); + if(c->socket<0) return c->socket; + + port=htons((unsigned short)(fui->port)); + if(ai->ai_addr->sa_family==AF_INET6) + ((struct sockaddr_in6*)(ai->ai_addr))->sin6_port=port; + else + ((struct sockaddr_in*)(ai->ai_addr))->sin_port=port; + + ret=frontier_connect(c->socket,ai->ai_addr,ai->ai_addrlen,c->connect_timeout_secs); + + if(ret!=FRONTIER_OK) + { + frontier_socket_close(c->socket); + c->socket=-1; + } + + c->cur_ai=ai; + + return ret; +} + + +#define FN_REQ_BUF 8192 + +#define FN_RET_LEN(RET,LEN) \ + if(RET>=FN_REQ_BUF-LEN) \ + { \ + frontier_setErrorMsg(__FILE__,__LINE__,"request is bigger than %d bytes",FN_REQ_BUF); \ + return FRONTIER_EIARG; \ + }; \ + LEN+=RET + +static int get_url(FrontierHttpClnt *c,const char *url,int is_post) +{ + int ret; + int len; + char buf[FN_REQ_BUF]; + FrontierUrlInfo *fui_server; + char *http_method; + + http_method=is_post?"POST":"GET"; + + fui_server=c->server[c->serveri.cur]; + + bzero(buf,FN_REQ_BUF); + + len=0; + if(c->using_proxy) + { + ret=snprintf(buf,FN_REQ_BUF,"%s %s%s%s%s HTTP/1.0\r\nHost: %s\r\n",http_method,fui_server->url,*url?"/":"",url,c->url_suffix,fui_server->host); + } + else + { + ret=snprintf(buf,FN_REQ_BUF,"%s /%s%s%s%s HTTP/1.0\r\nHost: %s\r\n",http_method,fui_server->path,*url?"/":"",url,c->url_suffix,fui_server->host); + } + FN_RET_LEN(ret,len); + + // Would use 'Cache-Control: max-stale=0' but squid 2.6 series (at least + // 2.6STABLE13 and 2.6STABLE18, but not 2.7STABLE4) then sends just + // 'Cache-Control: max-stale' (without =0) upstream and messes up + // its parent. max-stale=1 is almost the same thing except for one + // second, and requires squid to return an error if server is down + // rather than returning stale data. This option has very little + // effect, but it gets sent upstream to frontier server which now + // can send back a corresponding stale-if-error header which does + // tell squid 2.7 or later to return a 504 error if the origin server + // can't be reached when a cached item is stale. frontier_client + // couldn't handle that before this header was added, so this allows + // a compatible upgrade + // Leave \r\n off because we'll be adding it below + ret=snprintf(buf+len,FN_REQ_BUF-len, + "X-Frontier-Id: %s\r\nCache-Control: max-stale=1",c->frontier_id); + FN_RET_LEN(ret,len); + + // POST is always no-cache + if(is_post||(c->refresh_flag==2)) + { + ret=snprintf(buf+len,FN_REQ_BUF-len,",no-cache\r\n"); + } + else if(c->refresh_flag==1) + { + // Cache-control: max-age=0 allows If-Modified-Since to re-validate cache, + // which can be much less stress on servers than no-cache. + ret=snprintf(buf+len,FN_REQ_BUF-len,",max-age=0\r\n"); + } + else if((c->refresh_flag==0)&&(c->max_age>=0)) + { + ret=snprintf(buf+len,FN_REQ_BUF-len,",max-age=%d\r\n",c->max_age); + } + else + { + ret=snprintf(buf+len,FN_REQ_BUF-len,"\r\n"); + } + FN_RET_LEN(ret,len); + +#ifdef PERSISTCONNECTION + ret=snprintf(buf+len,FN_REQ_BUF-len,"Connection: keep-alive\r\n\r\n"); +#else + ret=snprintf(buf+len,FN_REQ_BUF-len,"\r\n"); +#endif + FN_RET_LEN(ret,len); + + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"request <%s>",buf); + + ret=frontier_write(c->socket,buf,len,c->write_timeout_secs,c->cur_ai); + if(ret<0) return ret; + return FRONTIER_OK; + } + + +static int read_connection(FrontierHttpClnt *c) + { + int ret; + int tot=0; + char buf[FN_REQ_BUF]; + + // clear out values from previous query (if any) + c->content_length=-1; + c->age=-1; + + // Read status line + ret=read_line(c,buf,FN_REQ_BUF); + if(ret<=0) return ret; + tot=ret; + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"status line <%s>",buf); + if((strncmp(buf,"HTTP/1.0 ",9)!=0)&&(strncmp(buf,"HTTP/1.1 ",9)!=0)) + { + frontier_setErrorMsg(__FILE__,__LINE__,"bad HTTP response (%s) proxy=%s server=%s", + buf,frontierHttpClnt_curproxyname(c),frontierHttpClnt_curservername(c)); + return FRONTIER_EPROTO; + } + + if((buf[9]=='5')||strncmp(&buf[9],"404",3)==0) + { + /* 5xx or 404 HTTP error codes indicate server error */ + frontier_setErrorMsg(__FILE__,__LINE__,"server error (%s) proxy=%s server=%s", + buf,frontierHttpClnt_curproxyname(c),frontierHttpClnt_curservername(c)); + return FRONTIER_ESERVER; + } + + if(strncmp(&buf[9],"200",3)!=0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"bad response (%s) proxy=%s server=%s", + buf,frontierHttpClnt_curproxyname(c),frontierHttpClnt_curservername(c)); + return FRONTIER_EPROTO; + } + + do + { + ret=read_line(c,buf,FN_REQ_BUF); + if(ret<0) return ret; + tot+=ret; + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"got line <%s>",buf); + /* look for "content-length" header */ +#define CLHEAD "content-length: " +#define CLLEN (sizeof(CLHEAD)-1) + if((c->content_length==-1)&&(ret>CLLEN)&&(strncasecmp(buf,CLHEAD,CLLEN)==0)) + c->content_length=atoi(buf+CLLEN); +#undef CLHEAD +#undef CLLEN + /* look for "age" header */ +#define AGEHEAD "age: " +#define AGELEN (sizeof(AGEHEAD)-1) + else if((c->age==-1)&&(ret>AGELEN)&&(strncasecmp(buf,AGEHEAD,AGELEN)==0)) + c->age=atoi(buf+AGELEN); +#undef AGEHEAD +#undef AGELEN + }while(*buf); + + return tot; + } + + +int frontierHttpClnt_open(FrontierHttpClnt *c) + { + int ret; + + ret=open_connection(c); + return ret; + } + + +int frontierHttpClnt_get(FrontierHttpClnt *c,const char *url) + { + return frontierHttpClnt_post(c,url,0); + } + + +int frontierHttpClnt_post(FrontierHttpClnt *c,const char *url,const char *body) + { + int ret; + int len=body?strlen(body):0; + int try=0; + + for(try=0;try<2;try++) + { + ret=get_url(c,url,0); + if(ret) return ret; + + if(len>0) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"body (%d bytes): [\n%s\n]",len,body); + ret=frontier_write(c->socket,body,len,c->write_timeout_secs,c->cur_ai); + if(ret<0) return ret; + } + + ret=read_connection(c); + if(ret==0) + { + if(try==0) + { + /*An empty response can happen on persisted connections after*/ + /*long waits. Close, reopen, and retry.*/ + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"got empty response, re-connecting and retrying"); + frontierHttpClnt_close(c); + if(frontierHttpClnt_open(c)==FRONTIER_OK)continue; + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"re-connect failed"); + } + frontier_setErrorMsg(__FILE__,__LINE__,"empty response from server proxy=%s server=%s", + frontierHttpClnt_curproxyname(c),frontierHttpClnt_curservername(c)); + return FRONTIER_ENETWORK; + } + if((ret==FRONTIER_ESYS)&&(try==0)) + { + /*System error 104 Connection reset by peer has been seen on*/ + /*heavily-loaded localhost squids (on a node with 8 cores).*/ + /*Close, reopen, and retry.*/ + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"Retrying after system error: %s",frontier_getErrorMsg()); + frontierHttpClnt_close(c); + if(frontierHttpClnt_open(c)==FRONTIER_OK)continue; + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"re-connect failed"); + } + /*if there was no continue, don't try again*/ + break; + } + + if(ret>0)return FRONTIER_OK; + return ret; + } + + +int frontierHttpClnt_read(FrontierHttpClnt *c,char *buf,int buf_len) + { + int ret; + int available; + +#ifdef PERSISTCONNECTION + if(c->content_length==0) + /* no more data to read for this url */ + return 0; +#endif + + if(c->data_pos+1>c->data_size) + { + ret=http_read(c); + if(ret<=0) return ret; + } + + available=c->data_size-c->data_pos; + available=buf_len>available?available:buf_len; +#ifdef PERSISTCONNECTION + if(c->content_length>0) + { + if(available>c->content_length) + { + frontier_setErrorMsg(__FILE__,__LINE__,"More data available (%d bytes) than Content-Length (%d bytes) says should be there",available,c->content_length); + return -1; + } + c->content_length-=available; + } + c->total_length+=available; +#endif + bcopy(c->buf+c->data_pos,buf,available); + c->data_pos+=available; + +#if 0 + { + /* print the beginning of the data in the block */ + char savech; + int lineno; + char *p = buf; + for (lineno = 0; lineno < 15; lineno++) + { + if ((lineno+1) * 40 >= available) + { + break; + } + savech = p[40]; + p[40] = '\0'; + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"data [%s]",p); + p[40] = savech; + p+=40; + } + if (lineno == 15) + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"(more data not logged)"); + } +#endif + + return available; + } + + +void frontierHttpClnt_close(FrontierHttpClnt *c) + { +#ifdef PERSISTCONNECTION + if(c->content_length<0) + { + /*there was no Content-Length header or this function was called twice*/ + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"closing persistent connection"); + /*note that "Proxy-Connection: close" was probably also set*/ + frontier_socket_close(c->socket); + c->socket=-1; + } + else if(c->total_length>=FRONTIER_MAX_PERSIST_SIZE) + { + /*squid inconsistently drops persistent connections after large objects + (at least in squid2.6STABLE13, it drops after those that were cache + MISSes upstream when the object initially loaded), so make it consistent + and close after every large object. This helps with load balancing.*/ + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"closing persistent connection after large object"); + frontier_socket_close(c->socket); + c->socket=-1; + } + else + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"persisting connection s=%d",c->socket); + } + c->total_length=0; +#else + frontier_socket_close(c->socket); + c->socket=-1; +#endif + c->data_pos=0; + c->data_size=0; + c->content_length=-1; + } + +void frontierHttpClnt_drop(FrontierHttpClnt *c) + { + if(c->socket!=-1) + { + // force the connection to drop + frontier_socket_close(c->socket); + c->socket=-1; + } + } + +void frontierHttpClnt_delete(FrontierHttpClnt *c) + { + int i; + + if(!c) return; + + frontierHttpClnt_drop(c); + + for(i=0;i<(sizeof(c->server)/sizeof(c->server[0])); i++) + frontier_DeleteUrlInfo(c->server[i]); + for(i=0;i<(sizeof(c->proxy)/sizeof(c->proxy[0])); i++) + frontier_DeleteUrlInfo(c->proxy[i]); + + if(c->frontier_id) frontier_mem_free(c->frontier_id); + + frontier_mem_free(c); + } + +void frontierHttpClnt_clear(FrontierHttpClnt *c) + { + // set so if there is a next time, it will reset the proxy & server lists + c->proxyi.whenreset=1; + c->serveri.whenreset=1; + } + +void frontierHttpClnt_resetwhenold(FrontierHttpClnt *c) + { + int i; + time_t now; + char nowbuf[26]; + + now=time(0); + if(c->proxyi.whenreset==0) + { + /*first time*/ + c->proxyi.whenreset=now; + c->serveri.whenreset=now; + return; + } + + if((now-c->proxyi.whenreset)proxyi.whenreset=now; + + if(c->socket!=-1) + { + /*close persisting connection*/ + frontier_socket_close(c->socket); + c->socket=-1; + } + + if(((c->proxyi.num_balanced>0)&&(c->proxyi.cur>=c->proxyi.num_balanced))|| + (c->proxyi.cur>0)) + { + //print warning if not in the first proxy group anymore + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"Resetting the proxy list at %s",frontier_str_now(nowbuf)); + } + + c->proxyi.cur=c->proxyi.first=0; + for(i=0;iproxyi.total;i++) + frontier_FreeAddrInfo(c->proxy[i]); + + if((now-c->serveri.whenreset)serveri.whenreset=now; + + if((c->serveri.num_balanced==0)&&(c->serveri.cur>0)) + { + //print warning if not in the first server group anymore + frontier_log(FRONTIER_LOGLEVEL_WARNING,__FILE__,__LINE__,"Resetting the server list at %s",frontier_str_now(nowbuf)); + } + + c->serveri.cur=c->serveri.first=0; + for(i=0;iserveri.total;i++) + frontier_FreeAddrInfo(c->server[i]); + + return; + } + +int frontierHttpClnt_getContentLength(FrontierHttpClnt *c) + { + return c->content_length; + } + +int frontierHttpClnt_getCacheAgeSecs(FrontierHttpClnt *c) + { + return c->age; + } + +// Advance to the next host in the current host group. +// Non-zero "persisting" means that the connection is persisting so no +// shuffling happens. +static int shufflehostgroup(FrontierHostsInfo *fhi,int persisting) + { + if(persisting||(fhi->cur>=fhi->total)) + { + /*don't shuffle if connection is persisting or there's no more hosts*/ + if(fhi->cur>=fhi->total) + return(-1); + return fhi->cur; + } + if((fhi->num_balanced>0)&&(fhi->curnum_balanced)) + { + // Randomize start if balancing was selected and at least two good hosts. + // Ignore the possibility that the addresses may include round-robin + // when balancing. + // Note that the different hosts may be of different ip families, + // and that's OK. Since they are separate names, the hosts may even + // contain different sets of families, eg. one ipv4 only and one + // dual stack. Each host will independently advance from their + // preferred to non-preferred family on errors. + int i,numgood=0; + for(i=0;inum_balanced;i++) + if(!fhi->hosts[i]->fai->haderror) + numgood++; + // Only need to shuffle if there's more than one good one. + if(numgood>1) + { + fhi->first=rand_r(&fhi->rand_seed)%numgood; + // skip the ones that had an error + for(i=0;i<=fhi->first;i++) + if(fhi->hosts[i]->fai->haderror) + fhi->first++; + fhi->cur=fhi->first; + } + } + else + { + // not in client-balancing group + // select next good round-robin address if any + int curfamily=0; + FrontierUrlInfo *fui=fhi->hosts[fhi->cur]; + FrontierAddrInfo *startfai=fui->fai; + do + { + // Shuffle only within the current ip family. fui->fai->ai may be + // unresolved here (that is, zero), but if fui->fai->next is non-zero + // that means round-robin and then it must be resolved. So don't need + // to check for ai non-zero on next. + // A bad side effect of this is that if there's a machine in the list + // that only has a non-preferred address working, it will not get + // any traffic. + if (fui->fai->ai!=0) + curfamily=fui->fai->ai->ai_family; + if (((fui->fai=fui->fai->next)==0)||(curfamily!=fui->fai->ai->ai_family)) + fui->fai=fui->firstfaiinfamily; + } while(fui->fai->haderror&&(fui->fai!=startfai)); + fui->lastfai=fui->fai; + } + return fhi->cur; + } + +int frontierHttpClnt_shuffleproxygroup(FrontierHttpClnt *c) + { + return shufflehostgroup(&c->proxyi,c->socket!=-1); + } + +int frontierHttpClnt_shuffleservergroup(FrontierHttpClnt *c) + { + return shufflehostgroup(&c->serveri,c->socket!=-1); + } + +// defined below, but used in resethostgroup +static int nexthost(FrontierHostsInfo *fhi,int curhaderror); + +// Reset the current host group so it can be used again. +// Non-zero "tobeginning" means to go to the very beginning of the list of +// multiple hosts; for proxies, that is when using loadbalance=proxies, +// and for servers, that is always (the latter because the retry strategy +// does not call for ever resetting to the beginning of an individual +// server group). Zero "tobeginning" means to instead go the beginning +// of the current IP family in a round-robin proxy. +static int resethostgroup(FrontierHostsInfo *fhi,int tobeginning) + { + FrontierUrlInfo *fui; + if(tobeginning) + { + // reset to the beginning of the host list + fhi->cur=fhi->first; + if(fhi->cur>=fhi->total) + return(-1); + fui=fhi->hosts[fhi->cur]; + } + else + { + // reset to the first address in a round-robin in the same family + fui=fhi->hosts[fhi->cur]; + fui->fai=fui->lastfai=fui->firstfaiinfamily; + } + if(fui->fai->haderror) + { + /*find one without an error*/ + return(nexthost(fhi,0)); + } + return(fhi->cur); + } + +int frontierHttpClnt_resetproxygroup(FrontierHttpClnt *c) + { + FrontierHostsInfo *fhi=&c->proxyi; + int tobeginning=0; + if(fhi->total==0) + { + /*no proxies are defined*/ + return(-1); + } + if(c->socket!=-1) + { + /*close persisting connection*/ + frontier_socket_close(c->socket); + c->socket=-1; + } + if((fhi->num_balanced>0)&&(fhi->curnum_balanced)) + tobeginning=1; + return(resethostgroup(fhi,tobeginning)); + } + +int frontierHttpClnt_resetserverlist(FrontierHttpClnt *c) + { + FrontierHostsInfo *fhi=&c->serveri; + if(c->socket!=-1) + { + /*close persisting connection*/ + frontier_socket_close(c->socket); + c->socket=-1; + } + return(resethostgroup(fhi,1)); + } + +int frontierHttpClnt_usinglastproxyingroup(FrontierHttpClnt *c) + { + FrontierHostsInfo *fhi=&c->proxyi; + FrontierUrlInfo *fui=fhi->hosts[fhi->cur]; + FrontierAddrInfo *nextfai; + int nexthost; + int curfamily=0; + if((fhi->num_balanced>0)&&(fhi->curnum_balanced)) + { + // using client load balancing + nexthost=fhi->cur+1; + if(nexthost==fhi->num_balanced) + nexthost=0; + if(nexthost!=fhi->first) + return 0; + return 1; + } + // Treat addresses in a different ip family as being a different group. + // Since this function is only for proxies, fai->ai is normally always + // nonzero, but it might be zero if the DNS lookup failed. + if(fui->fai->ai==0) + return 1; + curfamily=fui->fai->ai->ai_family; + if (((nextfai=fui->fai->next)==0)||(curfamily!=nextfai->ai->ai_family)) + nextfai=&fui->firstfai; + if(fui->lastfai!=nextfai) + return 0; + return 1; + } + +// Advance to the next host that has not been flagged as having had an error, +// whether in the current group or the next group. Non-zero "curhaderror" +// means that the current host had an error, so flag it as such. +static int nexthost(FrontierHostsInfo *fhi,int curhaderror) + { + FrontierUrlInfo *fui=fhi->hosts[fhi->cur]; + int curfamily=0; + // Note that if the host name has not been resolved because this is for + // a server and we're going through a proxy, fui->fai will still be a + // valid FrontierAddrInfo, it will just have ai=0 and next=0. + if(curhaderror) + fui->fai->haderror=1; + // advance the round-robin if there is any other address in the same family + if (fui->fai->ai!=0) + curfamily=fui->fai->ai->ai_family; + if (((fui->fai=fui->fai->next)==0)||(curfamily!=fui->fai->ai->ai_family)) + fui->fai=fui->firstfaiinfamily; + if(fui->lastfai==fui->fai) + { + /*end of this family in the round-robin, see if there's another family*/ + while(((fui->fai=fui->fai->next)!=0)&&(curfamily==fui->fai->ai->ai_family)) + ; + if(fui->fai!=0) + { + /*another family was found, advance to it*/ + fui->lastfai=fui->firstfaiinfamily=fui->fai; + } + else + { + fui->fai=fui->lastfai; + /*end of round-robin, advance through host list*/ + fhi->cur++; + if(fhi->num_balanced>0) + { + if(fhi->cur==fhi->num_balanced) + /*wrap around when reach the last balanced proxy*/ + fhi->cur=0; + if(fhi->cur==fhi->first) + { + /*done with balancing, set to the non-balanced ones, if any*/ + fhi->cur=fhi->num_balanced; + } + } + } + } + if(fhi->cur>=fhi->total) + { + /* Exhausted list. For each host in which all addresses have had + errors, clear the errors for possible next try. */ + int i; + for(i=0;itotal;i++) + { + FrontierAddrInfo *fai; + int anygood=0; + for(fai=&fhi->hosts[i]->firstfai;fai!=0;fai=fai->next) + { + if(!fai->haderror) + anygood=1; + if(inum_balanced) + break; // other round-robin addresses are ignored when balancing + } + if(!anygood) + { + for(fai=&fhi->hosts[i]->firstfai;fai!=0;fai=fai->next) + fai->haderror=0; + } + } + return(-1); + } + if(((fui=fhi->hosts[fhi->cur])->fai!=0)&&fui->fai->haderror) + /*this one had an error, choose another*/ + return(nexthost(fhi,0)); + return(fhi->cur); + } + +int frontierHttpClnt_nextproxy(FrontierHttpClnt *c,int curhaderror) + { + if(c->socket!=-1) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"closing persistent connection while choosing next proxy"); + frontier_socket_close(c->socket); + c->socket=-1; + } + return(nexthost(&c->proxyi,curhaderror)); + } + +int frontierHttpClnt_nextserver(FrontierHttpClnt *c,int curhaderror) + { + if(!c->using_proxy&&(c->socket!=-1)) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"closing persistent connection while choosing next server"); + frontier_socket_close(c->socket); + c->socket=-1; + } + return(nexthost(&c->serveri,curhaderror)); + } + +// return the current hostname for debug messages +static char *curhostname(FrontierHostsInfo *fhi) + { + FrontierUrlInfo *fui; + char *buf; + int len; + + if (fhi->cur>=fhi->total) + return ""; + + fui=fhi->hosts[fhi->cur]; + buf=fhi->debugbuf; + len=sizeof(fhi->debugbuf); + + strncpy(buf,fui->host,len); + if(fui->fai->ai!=0) + { + int n=strlen(fui->host); + buf+=n; + len-=n; + snprintf(buf,len,"[%s]",frontier_ipaddr(fui->fai->ai->ai_addr)); + buf[len-1]='\0'; + } + + return fhi->debugbuf; + } + +char *frontierHttpClnt_curproxyname(FrontierHttpClnt *c) + { + return(curhostname(&c->proxyi)); + } + +char *frontierHttpClnt_curservername(FrontierHttpClnt *c) + { + return(curhostname(&c->serveri)); + } + +char *frontierHttpClnt_curserverpath(FrontierHttpClnt *c) + { + FrontierHostsInfo *fhi = &c->serveri; + if (fhi->curtotal) + return fhi->hosts[fhi->cur]->path; + return ""; + } + +char *frontierHttpClnt_myipaddr(FrontierHttpClnt *c) + { + // allocate a size big enough for ipv6, which will also work for ipv4 + struct sockaddr_in6 sockaddrbuf; + socklen_t namelen=sizeof(sockaddrbuf); + // re-purpose the server debug buffer + char *buf=c->serveri.debugbuf; + if(getsockname(c->socket, (struct sockaddr *)&sockaddrbuf, &namelen)<0) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"cannot get sockname for socket %d: %s",c->socket,strerror(errno)); + return NULL; + } + strncpy(buf,frontier_ipaddr((struct sockaddr *)&sockaddrbuf),sizeof(c->serveri.debugbuf)); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"my ip addr: %s",buf); + return(buf); + } + +void frontierHttpClnt_setNumBalancedProxies(FrontierHttpClnt *c,int num) + { + c->proxyi.num_balanced=num; + if(c->proxyi.rand_seed==0) + initrand(c); + } + +void frontierHttpClnt_setBalancedServers(FrontierHttpClnt *c) + { + c->serveri.num_balanced=c->serveri.total; + if(c->serveri.rand_seed==0) + initrand(c); + } + diff --git a/http/fn-htclient.h b/http/fn-htclient.h new file mode 100644 index 0000000..ccb1e0d --- /dev/null +++ b/http/fn-htclient.h @@ -0,0 +1,150 @@ +/* + * frontier client http interface header + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#ifndef __HEADER_HTTP_H_FN_HTCLIENT_H +#define __HEADER_HTTP_H_FN_HTCLIENT_H + +#include +#include +#include + +#include "frontier_client/frontier.h" +#include "frontier_client/frontier_error.h" + +char *frontier_ipaddr(const struct sockaddr *serv_addr); +int frontier_socket(sa_family_t family); +void frontier_socket_close(int s); +int frontier_connect(int s,const struct sockaddr *serv_addr,socklen_t addrlen,int timeoutsecs); +int frontier_write(int s,const char *buf,int len,int timeoutsecs,struct addrinfo *addr); +int frontier_read(int s, char *buf,int size,int timeoutsecs,struct addrinfo *addr); + + +struct s_FrontierAddrInfo + { + struct s_FrontierAddrInfo *next; + struct addrinfo *ai; + int haderror; + }; +typedef struct s_FrontierAddrInfo FrontierAddrInfo; + +struct s_FrontierUrlInfo + { + char *url; + char *proto; + char *host; + int port; + char *path; + struct addrinfo *ai; + FrontierAddrInfo firstfai; + FrontierAddrInfo *firstfaiinfamily; // first in the current ip family + FrontierAddrInfo *fai; + FrontierAddrInfo *lastfai; + }; +typedef struct s_FrontierUrlInfo FrontierUrlInfo; + +FrontierUrlInfo *frontier_CreateUrlInfo(const char *url,int *ec); +int frontier_resolv_host(FrontierUrlInfo *fui,int preferipfamily,unsigned *rand_seedp); +void frontier_DeleteUrlInfo(FrontierUrlInfo *fui); +void frontier_FreeAddrInfo(FrontierUrlInfo *fui); + +#define FRONTIER_HTTP_DEBUG_BUF_SIZE 512 + +struct s_FrontierHostsInfo + { + FrontierUrlInfo **hosts; + int cur; + int total; + int num_balanced; + int first; + time_t whenreset; + char debugbuf[FRONTIER_HTTP_DEBUG_BUF_SIZE]; + unsigned rand_seed; + }; +typedef struct s_FrontierHostsInfo FrontierHostsInfo; + +#define FRONTIER_RESETPROXY_SECS (60*5) +#define FRONTIER_RESETSERVER_SECS (60*30) +#define FRONTIER_HTTP_BUF_SIZE (32*1024) +#define FRONTIER_MAX_PERSIST_SIZE (16*1024) + +struct s_FrontierHttpClnt + { + FrontierUrlInfo *proxy[FRONTIER_MAX_PROXYN]; + FrontierUrlInfo *server[FRONTIER_MAX_SERVERN]; + FrontierHostsInfo proxyi; + FrontierHostsInfo serveri; + + int refresh_flag; + int max_age; + int age; + int prefer_ip_family; + int using_proxy; + int content_length; + int total_length; + char *url_suffix; + char *frontier_id; + + int socket; + int connect_timeout_secs; + int read_timeout_secs; + int write_timeout_secs; + struct addrinfo *cur_ai; + + int err_code; + int data_size; + int data_pos; + char buf[FRONTIER_HTTP_BUF_SIZE]; + }; +typedef struct s_FrontierHttpClnt FrontierHttpClnt; + + +FrontierHttpClnt *frontierHttpClnt_create(int *ec); +int frontierHttpClnt_addServer(FrontierHttpClnt *c,const char *url); +int frontierHttpClnt_addProxy(FrontierHttpClnt *c,const char *url); +void frontierHttpClnt_setCacheRefreshFlag(FrontierHttpClnt *c,int refresh_flag); +void frontierHttpClnt_setCacheMaxAgeSecs(FrontierHttpClnt *c,int secs); +void frontierHttpClnt_setPreferIpFamily(FrontierHttpClnt *c,int ipfamily); +void frontierHttpClnt_setUrlSuffix(FrontierHttpClnt *c,char *suffix); +void frontierHttpClnt_setFrontierId(FrontierHttpClnt *c,const char *frontier_id); +void frontierHttpClnt_setConnectTimeoutSecs(FrontierHttpClnt *c,int timeoutsecs); +void frontierHttpClnt_setReadTimeoutSecs(FrontierHttpClnt *c,int timeoutsecs); +void frontierHttpClnt_setWriteTimeoutSecs(FrontierHttpClnt *c,int timeoutsecs); +int frontierHttpClnt_open(FrontierHttpClnt *c); +int frontierHttpClnt_get(FrontierHttpClnt *c,const char *url); +int frontierHttpClnt_post(FrontierHttpClnt *c,const char *url,const char *body); +int frontierHttpClnt_read(FrontierHttpClnt *c,char *buf,int buf_len); +void frontierHttpClnt_close(FrontierHttpClnt *c); +void frontierHttpClnt_drop(FrontierHttpClnt *c); +void frontierHttpClnt_delete(FrontierHttpClnt *c); +void frontierHttpClnt_clear(FrontierHttpClnt *c); +void frontierHttpClnt_resetwhenold(FrontierHttpClnt *c); +int frontierHttpClnt_getContentLength(FrontierHttpClnt *c); +int frontierHttpClnt_getCacheAgeSecs(FrontierHttpClnt *c); +int frontierHttpClnt_shuffleproxygroup(FrontierHttpClnt *c); +int frontierHttpClnt_shuffleservergroup(FrontierHttpClnt *c); +int frontierHttpClnt_resetproxygroup(FrontierHttpClnt *c); +int frontierHttpClnt_resetserverlist(FrontierHttpClnt *c); +int frontierHttpClnt_nextproxy(FrontierHttpClnt *c,int curhaderror); +int frontierHttpClnt_nextserver(FrontierHttpClnt *c,int curhaderror); +int frontierHttpClnt_usinglastproxyingroup(FrontierHttpClnt *c); +char *frontierHttpClnt_curproxyname(FrontierHttpClnt *c); +char *frontierHttpClnt_curservername(FrontierHttpClnt *c); +char *frontierHttpClnt_curserverpath(FrontierHttpClnt *c); +char *frontierHttpClnt_myipaddr(FrontierHttpClnt *c); +void frontierHttpClnt_setNumBalancedProxies(FrontierHttpClnt *c,int num); +void frontierHttpClnt_setBalancedServers(FrontierHttpClnt *c); + + +#endif //__HEADER_HTTP_H_FN_HTCLIENT_H diff --git a/http/fn-socket.c b/http/fn-socket.c new file mode 100644 index 0000000..44005fe --- /dev/null +++ b/http/fn-socket.c @@ -0,0 +1,278 @@ +/* + * frontier client socket interface + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// return ascii representation of ipv4 or ipv6 address, for log messages +// also add colon and port number if port number is not zero +// it is stored in a static buffer, overwritten by later calls +char *frontier_ipaddr(const struct sockaddr *serv_addr) + { + // addrbuf may contain extra brackets, colon, port number, and + // null terminator. 16 will more than cover it. + static char addrbuf[INET6_ADDRSTRLEN+16]; + in_port_t port; + char *p; + if (serv_addr->sa_family == AF_INET6) + { + addrbuf[0]='['; + addrbuf[1]='\0'; + inet_ntop(AF_INET6,&(((struct sockaddr_in6*)serv_addr)->sin6_addr),&addrbuf[1],INET6_ADDRSTRLEN); + p=strchr(addrbuf,'\0'); + *p++=']'; + *p='\0'; + port=ntohs(((struct sockaddr_in6*)serv_addr)->sin6_port); + } + else + { + addrbuf[0]='\0'; + inet_ntop(AF_INET,&(((struct sockaddr_in*)serv_addr)->sin_addr),addrbuf,INET_ADDRSTRLEN); + p=strchr(addrbuf,'\0'); + port=ntohs(((struct sockaddr_in*)serv_addr)->sin_port); + } + if(port!=0) + { + snprintf(p,&addrbuf[sizeof(addrbuf)-1]-p,":%d",(int)port); + addrbuf[sizeof(addrbuf)-1]='\0'; // in case of overflow + } + return addrbuf; + } + +//static int total_socket=0; + +int frontier_socket(sa_family_t family) + { + int s; + int flags; + + s=socket(family,SOCK_STREAM,0); + if(s<0) goto err; + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"new socket s=%d",s); + + flags=fcntl(s,F_GETFL); + if(flags<0) goto err; + + flags=flags|O_NONBLOCK; + flags=fcntl(s,F_SETFL,(long)flags); + if(flags<0) goto err; + goto ok; + +err: + if(s>=0) frontier_socket_close(s); + frontier_setErrorMsg(__FILE__,__LINE__,"system error %d: %s",errno,strerror(errno)); + return FRONTIER_ESYS; + +ok: + //++total_socket; + //printf("so %d\n",total_socket); + return s; + } + + +void frontier_socket_close(int s) + { + close(s); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"socket s=%d closed",s); + //--total_socket; + //printf("sc %d\n",total_socket); + } + +static int handle_connect_error(const struct sockaddr *serv_addr) + { + if(errno==ECONNREFUSED || errno==ENETUNREACH || errno==EHOSTUNREACH) + { + frontier_setErrorMsg(__FILE__,__LINE__,"network error on connect to %s: %s",frontier_ipaddr(serv_addr),strerror(errno)); + return FRONTIER_ECONNECT; + } + else + { + frontier_setErrorMsg(__FILE__,__LINE__,"system error %d on connect to %s: %s",errno,frontier_ipaddr(serv_addr),strerror(errno)); + return FRONTIER_ESYS; + } + } + +int frontier_connect(int s,const struct sockaddr *serv_addr,socklen_t addrlen,int timeoutsecs) + { + int ret; + struct pollfd pfd; + int val; + socklen_t s_len; + + ret=connect(s,serv_addr,addrlen); + if(ret==0) return FRONTIER_OK; + + /* ret<0 below */ + if(errno!=EINPROGRESS) + return(handle_connect_error(serv_addr)); + + /* non-blocking connect in progress here */ + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"connect s=%d addr %s waiting for response",s,frontier_ipaddr(serv_addr)); + pfd.fd=s; + pfd.events=POLLOUT; + do + { + pfd.revents=0; + ret=poll(&pfd,1,timeoutsecs*1000); + }while((ret<0)&&(errno==EINTR)); /*this loop is to support profiling*/ + if(ret<0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"system error %d on poll when connecting to %s: %s",errno,frontier_ipaddr(serv_addr),strerror(errno)); + return FRONTIER_ESYS; + } + if(ret==0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"connect to %s timed out after %d seconds",frontier_ipaddr(serv_addr),timeoutsecs); + return FRONTIER_ECONNECT; + } + + s_len=sizeof(val); + val=0; + ret=getsockopt(s,SOL_SOCKET,SO_ERROR,&val,&s_len); + if(ret<0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"system error %d on getsockopt when connecting to %s: %s",errno,frontier_ipaddr(serv_addr),strerror(errno)); + return FRONTIER_ESYS; + } + if(val) + { + errno=val; + return(handle_connect_error(serv_addr)); + } + + if(!(pfd.revents&POLLOUT)) + { + frontier_setErrorMsg(__FILE__,__LINE__,"inconsistent result from connect poll on fd %d",s); + return FRONTIER_EUNKNOWN; + } + + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"connected, s=%d .",s); + return FRONTIER_OK; + } + + +static int socket_write(int s,const char *buf, int len, int timeoutsecs,struct addrinfo *addr) + { + int ret; + struct pollfd pfd; + + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"write request %d bytes",len); + + pfd.fd=s; + pfd.events=POLLOUT; + pfd.revents=0; + ret=poll(&pfd,1,timeoutsecs*1000); + if(ret<0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"system error %d on poll: %s",errno,strerror(errno)); + return FRONTIER_ESYS; + } + if(ret==0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"write to %s timed out after %d seconds", + frontier_ipaddr(addr->ai_addr),timeoutsecs); + return FRONTIER_ENETWORK; + } + + if(!(pfd.revents&POLLOUT)) + { + frontier_setErrorMsg(__FILE__,__LINE__,"inconsistent result from write poll on fd %d",s); + return FRONTIER_EUNKNOWN; + } + + ret=send(s,buf,len,0); + if(ret>=0) return ret; + + if(errno==ECONNRESET) + { + frontier_setErrorMsg(__FILE__,__LINE__,"connection reset by peer"); + return FRONTIER_ENETWORK; + } + + frontier_setErrorMsg(__FILE__,__LINE__,"system error %d: %s",errno,strerror(errno)); + return FRONTIER_ESYS; + } + + +int frontier_write(int s,const char *buf, int len, int timeoutsecs,struct addrinfo *addr) + { + int ret; + int total; + int repeat; + + repeat=3; // XXX repeat write 3 times (if no error) + + total=0; + while(repeat--) + { + ret=socket_write(s,buf+total,len-total,timeoutsecs,addr); + if(ret<0) return ret; + total+=ret; + if(total==len) return total; + } + + frontier_setErrorMsg(__FILE__,__LINE__,"failed to write over network"); + return FRONTIER_ENETWORK; + } + + + +int frontier_read(int s, char *buf, int size, int timeoutsecs,struct addrinfo *addr) + { + int ret; + struct pollfd pfd; + + pfd.fd=s; + pfd.events=POLLIN; + do + { + pfd.revents=0; + ret=poll(&pfd,1,timeoutsecs*1000); + }while((ret<0)&&(errno==EINTR)); /*this loop is to support profiling*/ + if(ret<0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"system error %d on poll: %s",errno,strerror(errno)); + return FRONTIER_ESYS; + } + if(ret==0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"read from %s timed out after %d seconds", + frontier_ipaddr(addr->ai_addr),timeoutsecs); + return FRONTIER_ENETWORK; + } + + if(!(pfd.revents&POLLIN)) + { + frontier_setErrorMsg(__FILE__,__LINE__,"inconsistent result from poll on fd %d",s); + return FRONTIER_EUNKNOWN; + } + + ret=recv(s,buf,size,0); + if(ret>=0) return ret; + + frontier_setErrorMsg(__FILE__,__LINE__,"system error %d: %s",errno,strerror(errno)); + return FRONTIER_ESYS; + } + diff --git a/http/fn-urlparse.c b/http/fn-urlparse.c new file mode 100644 index 0000000..94990d5 --- /dev/null +++ b/http/fn-urlparse.c @@ -0,0 +1,307 @@ +/* + * frontier client url parsing + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#include +#include "../fn-internal.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HOST_BUF_SIZE 256 + + +FrontierUrlInfo *frontier_CreateUrlInfo(const char *url,int *ec) + { + FrontierUrlInfo *fui; + const char *p; + int i; + char hostbuf[HOST_BUF_SIZE+1]; + + fui=frontier_mem_alloc(sizeof(FrontierUrlInfo)); + if(!fui) + { + *ec=FRONTIER_EMEM; + FRONTIER_MSG(*ec); + goto err; + } + bzero(fui,sizeof(FrontierUrlInfo)); + fui->fai=fui->lastfai=fui->firstfaiinfamily=&fui->firstfai; + + fui->url=frontier_str_copy(url); + if(!fui->url) + { + *ec=FRONTIER_EMEM; + FRONTIER_MSG(*ec); + goto err; + } + + p=url; + if(strncmp(url,"http://",7)==0) + p+=7; + else if(strstr(url, "://")!=NULL) + { + frontier_setErrorMsg(__FILE__,__LINE__,"config error: unsupported proto in url %s",url); + *ec=FRONTIER_ECFG; + goto err; + } + fui->proto=frontier_str_copy("http"); // No other protocols yet + i=0; + + if (*p=='[') + { + // Square brackets around IPv6 IP address + p++; + while(*p && (*p!=']') && (i=HOST_BUF_SIZE)) + { + frontier_setErrorMsg(__FILE__,__LINE__,"config error: bad host name in url %s",url); + *ec=FRONTIER_ECFG; + goto err; + } + + hostbuf[i]='\0'; + fui->host=frontier_str_copy(hostbuf); + + if(*p==':') + { + ++p; + fui->port=atoi(p); + while(*p && (*p>='0') && (*p<='9')) ++p; + } + else + fui->port=80; + + if(fui->port<=0 || fui->port>=65534) + { + frontier_setErrorMsg(__FILE__,__LINE__,"config error: bad port number in url %s",url); + *ec=FRONTIER_ECFG; + goto err; + } + + if(*p) + { + if (*p!='/') + { + frontier_setErrorMsg(__FILE__,__LINE__,"config error: bad url %s",url); + *ec=FRONTIER_ECFG; + goto err; + } + fui->path=frontier_str_copy(p+1); + } + + *ec=FRONTIER_OK; + return fui; + +err: + frontier_DeleteUrlInfo(fui); + return NULL; + } + +void frontier_FreeAddrInfo(FrontierUrlInfo *fui) + { + if(fui->ai) + { + // this frees all addrinfo structures in the round-robin chain + freeaddrinfo(fui->ai); + fui->ai=0; + } + while(fui->firstfai.next) + { + fui->fai=fui->firstfai.next->next; + frontier_mem_free(fui->firstfai.next); + fui->firstfai.next=fui->fai; + } + bzero(&fui->firstfai,sizeof(FrontierAddrInfo)); + fui->fai=fui->lastfai=fui->firstfaiinfamily=&fui->firstfai; + } + +int frontier_resolv_host(FrontierUrlInfo *fui,int preferipfamily,unsigned *rand_seedp) + { + struct addrinfo hints; + int ret; + struct addrinfo *ai; + FrontierAddrInfo *fai; + sa_family_t family; + int ipfamily; + int didpreferred,scanagain; + int numipv4,numipv6,numaddrs; + int i,starti; + + if(fui->fai->ai) + return FRONTIER_OK; + + if (preferipfamily!=0) + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"getting addr info for %s, prefer ipv%d",fui->host,preferipfamily); + + bzero(&hints,sizeof(struct addrinfo)); + + hints.ai_family=PF_UNSPEC; + hints.ai_socktype=SOCK_STREAM; + hints.ai_protocol=0; + + frontier_FreeAddrInfo(fui); + ret=getaddrinfo(fui->host,NULL,&hints,&(fui->ai)); + if(ret) + { + if(ret==EAI_SYSTEM) + { + frontier_setErrorMsg(__FILE__,__LINE__,"host name %s problem: %s",fui->host,strerror(errno)); + } + else + { + frontier_setErrorMsg(__FILE__,__LINE__,"host name %s problem: %s",fui->host,gai_strerror(ret)); + } + return FRONTIER_ENETWORK; + } + + // for each ai after the first, allocate an fai + // (it's after the first because one fai is pre-allocated) + fai=&fui->firstfai; + ai=fui->ai; + numipv4=0; + numipv6=0; + numaddrs=0; + while(ai) + { + if(ai->ai_family==AF_INET) + numipv4++; + else if(ai->ai_family==AF_INET6) + numipv6++; + else + { + // shouldn't happen, but just in case + ai=ai->ai_next; + continue; + } + ai=ai->ai_next; + if(numaddrs++==0) + continue; // skip allocating for preallocated one + FrontierAddrInfo *nextfai=frontier_mem_alloc(sizeof(FrontierAddrInfo)); + if(!fai) + { + FRONTIER_MSG(FRONTIER_EMEM); + return FRONTIER_EMEM; + } + bzero(nextfai,sizeof(FrontierAddrInfo)); + fai->next=nextfai; + fai=nextfai; + } + + fai=&fui->firstfai; + + // assign an ai to each fai, preferred family first + + ai=fui->ai; + if(preferipfamily==4) + family=AF_INET; + else if(preferipfamily==6) + family=AF_INET6; + else + family=ai->ai_family; // prefer the family of the first one + + didpreferred=0; + while(1) // do once for each family + { + if(family==AF_INET) + { + numaddrs=numipv4; + ipfamily=4; + } + else if(family==AF_INET6) + { + numaddrs=numipv6; + ipfamily=6; + } + else + { + numaddrs=0; + ipfamily=0; + } + + scanagain=0; + i=0; + starti=0; + if(numaddrs>1) + { + // Randomize the starting address for this family + starti=rand_r(rand_seedp)%numaddrs; + if(starti>0) + { + while(iai_family==family) + i++; + ai=ai->ai_next; + } + scanagain=1; + } + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"%s: ipv%d random start address %d of %d",fui->host,ipfamily,starti+1,numaddrs); + } + while(1) // scan twice if starti was greater than 0 + { + while(ai&&(scanagain||(starti==0)||(iai_family==family) + { + i++; + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"%s: found ipv%d addr <%s>",fui->host,ipfamily,frontier_ipaddr(ai->ai_addr)); + fai->ai=ai; + fai=fai->next; + } + ai=ai->ai_next; + } + if(!scanagain) + break; + scanagain=0; + i=0; + ai=fui->ai; + } + if(didpreferred) + break; + didpreferred=1; + if(family==AF_INET) + family=AF_INET6; + else + family=AF_INET; + ai=fui->ai; + } + + return FRONTIER_OK; + } + +void frontier_DeleteUrlInfo(FrontierUrlInfo *fui) + { + if(!fui) return; + frontier_FreeAddrInfo(fui); + if(fui->url) frontier_mem_free(fui->url); + if(fui->proto) frontier_mem_free(fui->proto); + if(fui->host) frontier_mem_free(fui->host); + if(fui->path) frontier_mem_free(fui->path); + + frontier_mem_free(fui); + } + diff --git a/http/test-htclient.c b/http/test-htclient.c new file mode 100644 index 0000000..e13d4fd --- /dev/null +++ b/http/test-htclient.c @@ -0,0 +1,87 @@ +/* + * frontier client http subset main test program + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#include +#include +#include +#include + +#include +#include + + +#define B_SIZE 65536 + +int main(int argc,char **argv) + { + FrontierHttpClnt *clnt; + int ret; + char buf[B_SIZE]; + + if(argc!=5) + { + printf("Usage: %s server_url obj_name key_name key_value\n",argv[0]); + exit(1); + } + + clnt=frontierHttpClnt_create(); + if(!clnt) + { + printf("Error %s\n",frontier_getErrorMsg()); + exit(1); + } + + ret=frontierHttpClnt_addServer(clnt,argv[1]); + if(ret) + { + printf("Error %s\n",frontier_getErrorMsg()); + exit(1); + } + + ret=frontierHttpClnt_addProxy(clnt,"http://edge:8128"); + if(ret) + { + printf("Error %s\n",frontier_getErrorMsg()); + exit(1); + } + + frontierHttpClnt_setCacheRefreshFlag(clnt,0); + + bzero(buf,B_SIZE); + snprintf(buf,B_SIZE,"Frontier?type=%s:1&encoding=BLOB&%s=%s",argv[2],argv[3],argv[4]); + printf("Req <%s>\n",buf); + + ret=frontierHttpClnt_open(clnt,buf); + if(ret) + { + printf("Error %s\n",frontier_getErrorMsg()); + exit(1); + } + + bzero(buf,B_SIZE); + + printf("Data:\n"); + while((ret=frontierHttpClnt_read(clnt,buf,B_SIZE-1))>0) {printf("%s",buf); bzero(buf,B_SIZE);} + printf("End.\n"); + if(ret<0) + { + printf("Error %s\n",frontier_getErrorMsg()); + exit(1); + } + + frontierHttpClnt_delete(clnt); + + exit(0); + } diff --git a/include/frontier_client/FrontierException.hpp b/include/frontier_client/FrontierException.hpp new file mode 100644 index 0000000..96e6942 --- /dev/null +++ b/include/frontier_client/FrontierException.hpp @@ -0,0 +1,125 @@ +/* + * frontier client C++ API exceptions header + * + * Author: Sinisa Veseli + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ +# +#ifndef FRONTIER_EXCEPTION_HPP +#define FRONTIER_EXCEPTION_HPP + +#include +#include +#include + +extern "C" { +#include "frontier_client/frontier_error.h" +} + + + +namespace frontier { + + static const int FrontierErrorCode_OK = FRONTIER_OK; + static const int FrontierErrorCode_InvalidArgument = FRONTIER_EIARG; + static const int FrontierErrorCode_MemoryAllocationFailed = FRONTIER_EMEM; + static const int FrontierErrorCode_ConfigurationError = FRONTIER_ECFG; + static const int FrontierErrorCode_SystemError = FRONTIER_ESYS; + static const int FrontierErrorCode_UnknownError = FRONTIER_EUNKNOWN; + static const int FrontierErrorCode_NetworkProblem = FRONTIER_ENETWORK; + static const int FrontierErrorCode_ProtocolError = FRONTIER_EPROTO; + static const int FrontierErrorCode_ServerError = FRONTIER_ESERVER; + + // Generic exception. + class FrontierException : public std::exception { + private: + std::string _error; + int _errorCode; + public: + virtual ~FrontierException() throw() { } + FrontierException(const std::string& err = "", int errorCode = FrontierErrorCode_UnknownError) : _error(err), _errorCode(errorCode) { + std::string additionalInfo(frontier_getErrorMsg()); + if(additionalInfo != std::string("")) { + _error = _error + " (Additional Information: " + additionalInfo + ")"; + } + } + virtual std::string getError() const throw() { return error(); } + virtual int getErrorCode() const throw() { return _errorCode; } + virtual std::string error() const throw() { return _error; } + virtual const char* what() const throw() { return _error.c_str(); } + }; + + // Output. + inline std::ostream& operator<<(std::ostream& os, + const FrontierException& ex) { + os << ex.error(); + return os; + } + + // Runtime error. + class RuntimeError : public FrontierException { + public: + RuntimeError(const std::string& err = "", int errorCode = FrontierErrorCode_UnknownError) : FrontierException(err, errorCode) { } + }; + + // Configuration error. + class ConfigurationError : public FrontierException { + public: + ConfigurationError(const std::string& err = "") : FrontierException(err, FrontierErrorCode_ConfigurationError) { } + }; + + // Logic error. + class LogicError : public FrontierException { + public: + LogicError(const std::string& err = "", int errorCode = FrontierErrorCode_UnknownError) : FrontierException(err, errorCode) { } + }; + + // Invalid argument. + class InvalidArgument : public LogicError { + public: + InvalidArgument(const std::string& err = "") : LogicError(err, FrontierErrorCode_InvalidArgument) { } + }; + + // System error. + class SystemError : public FrontierException { + public: + SystemError(const std::string& err = "", int errorCode = FrontierErrorCode_SystemError) : FrontierException(err, errorCode) { } + }; + + // Memory allocation error. + class MemoryAllocationFailed : public SystemError { + public: + MemoryAllocationFailed(const std::string& err = "") : SystemError(err, FrontierErrorCode_MemoryAllocationFailed) { } + }; + + // System error. + class NetworkProblem : public FrontierException { + public: + NetworkProblem(const std::string& err = "") : FrontierException(err, FrontierErrorCode_NetworkProblem) { } + }; + + // Protocol error. + class ProtocolError : public FrontierException { + public: + ProtocolError(const std::string& err = "") : FrontierException(err, FrontierErrorCode_ProtocolError) { } + }; + + // Server error. + class ServerError : public FrontierException { + public: + ServerError(const std::string& err = "") : FrontierException(err, FrontierErrorCode_ServerError) { } + }; + +} // namespace frontier + +#endif // ifndef FRONTIER_EXCEPTION + + diff --git a/include/frontier_client/FrontierExceptionMapper.hpp b/include/frontier_client/FrontierExceptionMapper.hpp new file mode 100644 index 0000000..05a0220 --- /dev/null +++ b/include/frontier_client/FrontierExceptionMapper.hpp @@ -0,0 +1,38 @@ +/* + * frontier client C++ API exception mapper header + * + * Author: Sinisa Vaseli + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ +# +#ifndef FRONTIER_EXCEPTION_MAP_HPP +#define FRONTIER_EXCEPTION_MAP_HPP + +#include +#include +#include + +#include "frontier_client/FrontierException.hpp" + +namespace frontier { + + class FrontierExceptionMapper { + private: + FrontierExceptionMapper() {} + public: + static void throwException(int errorCode, const std::string& errorMessage); + }; + +} // namespace frontier + +#endif // ifndef FRONTIER_EXCEPTION_MAP_HPP + + diff --git a/include/frontier_client/frontier-cpp.h b/include/frontier_client/frontier-cpp.h new file mode 100644 index 0000000..8611fc7 --- /dev/null +++ b/include/frontier_client/frontier-cpp.h @@ -0,0 +1,278 @@ +/* + * frontier client C++ API header + * + * Author: Sergey Kosyakov + * + * $Header$ + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ +#ifndef __HEADER_H_FRONTIER_FRONTIER_CPP_H_ +#define __HEADER_H_FRONTIER_FRONTIER_CPP_H_ + +#include +#include +#include + +extern "C" + { +#include "frontier_client/frontier.h" + } + +namespace frontier{ + +enum encoding_t {BLOB}; + +class Session; + +class Request + { + protected: + friend class Session; + std::string obj_name; + encoding_t enc; + std::vector *v_key; + std::vector *v_val; + int is_meta; + + public: + Request(const std::string& name, + const encoding_t& encoding): + obj_name(name),enc(encoding),v_key(NULL),v_val(NULL),is_meta(0){}; + + virtual void addKey(const std::string& key,const std::string& value); + virtual ~Request(); + + static std::string encodeParam(const std::string &value); + // set the zip level of retrieved data + // level 0 is off, level 1 is fast, level 5 is normal, level 9 is best + // default is 5 + static void setRetrieveZipLevel(int level); + }; + + +class MetaRequest : public Request + { + public: + MetaRequest(const std::string& name, + const encoding_t& encoding):Request(name,encoding) {is_meta=1;} + virtual void addKey(const std::string&,const std::string&){} + virtual ~MetaRequest(){} + }; + + +int init(); +// loglevel can be "nolog", "error", "info" or "warning" (which are equivalent), +// or anything else (which is treated as "debug") +// each level includes all messages at lower levels +int init(const std::string& logfilename, const std::string& loglevel); + +// Enum sucks +typedef unsigned char BLOB_TYPE; +const BLOB_TYPE BLOB_BIT_NULL=(1<<7); + +const BLOB_TYPE BLOB_TYPE_NONE=BLOB_BIT_NULL; +const BLOB_TYPE BLOB_TYPE_BYTE=0; +const BLOB_TYPE BLOB_TYPE_INT4=1; +const BLOB_TYPE BLOB_TYPE_INT8=2; +const BLOB_TYPE BLOB_TYPE_FLOAT=3; +const BLOB_TYPE BLOB_TYPE_DOUBLE=4; +const BLOB_TYPE BLOB_TYPE_TIME=5; +const BLOB_TYPE BLOB_TYPE_ARRAY_BYTE=6; +const BLOB_TYPE BLOB_TYPE_EOR=7; + +const char *getFieldTypeName(BLOB_TYPE t); + +class Session; + +class AnyData + { + private: + friend class Session; + union + { + long long i8; + double d; + struct{char *p;unsigned int s;}str; + int i4; + float f; + char b; + } v; + int isNull; // I do not use "bool" here because of compatibility problems [SSK] + int type_error; + BLOB_TYPE t; // The data type + Session *sessionp; + int unused1; // for binary compatibility -- available for reuse + + int castToInt(); + long long castToLongLong(); + float castToFloat(); + double castToDouble(); + std::string* castToString(); + + public: + AnyData() + :isNull(0) + ,type_error(FRONTIER_OK) + ,t(BLOB_TYPE_NONE) + {sessionp=0;unused1=0;} + + long long getRawI8() const {return v.i8;} + double getRawD() const {return v.d;} + const char *getRawStrP() const {return v.str.p;} + unsigned getRawStrS() const {return v.str.s;} + int getRawI4() const {return v.i4;} + float getRawF() const {return v.f;} + char getRawB() const {return v.b;} + + inline void set(int i4){t=BLOB_TYPE_INT4;v.i4=i4;type_error=FRONTIER_OK;} + inline void set(long long i8){t=BLOB_TYPE_INT8;v.i8=i8;type_error=FRONTIER_OK;} + inline void set(float f){t=BLOB_TYPE_FLOAT;v.f=f;type_error=FRONTIER_OK;} + inline void set(double d){t=BLOB_TYPE_DOUBLE;v.d=d;type_error=FRONTIER_OK;} + inline void set(long long t,int /*time*/){t=BLOB_TYPE_TIME;v.i8=t;type_error=FRONTIER_OK;} + inline void set(unsigned int size,char *ptr){t=BLOB_TYPE_ARRAY_BYTE;v.str.s=size;v.str.p=ptr;type_error=FRONTIER_OK;} + inline void setEOR(){t=BLOB_TYPE_EOR;type_error=FRONTIER_OK;} + inline BLOB_TYPE type() const{return t;} + inline int isEOR() const{return (t==BLOB_TYPE_EOR);} + + inline int getInt(){if(isNull) return 0;if(t==BLOB_TYPE_INT4) return v.i4; return castToInt();} + inline long long getLongLong(){if(isNull) return 0; if(t==BLOB_TYPE_INT8 || t==BLOB_TYPE_TIME) return v.i8; return castToLongLong();} + inline float getFloat(){if(isNull) return 0.0; if(t==BLOB_TYPE_FLOAT) return v.f; return castToFloat();} + inline double getDouble(){if(isNull) return 0.0;if(t==BLOB_TYPE_DOUBLE) return v.d; return castToDouble();} + std::string* getString(); + void assignString(std::string *s); + void clean(); + ~AnyData(); + }; + + +class Session; + +class Connection + { + private: + friend class Session; + unsigned long channel; + + public: + + explicit Connection(const std::string& server_url="",const std::string* proxy_url=NULL); + + // This constructor allows initialization with multiple server/proxies. + explicit Connection(const std::list& serverUrlList, const std::list& proxyUrlList); + + virtual ~Connection(); + + // Set cache time to live for following requested objects. + // 1=short, 2=long, 3=forever. Default 2. + void setTimeToLive(int ttl); + + // Deprecated interface: 0 -> setTimeToLive(2), !0 -> setTimeToLive(1) + void setReload(int reload); + + // Set default parameters for later-created Connections. + // "logicalServer" is the default logical server URL. Any time + // that URL is requested, it will be ignored and instead any other + // servers or proxies specified will be used. "parameterList" is + // a concatenated list of parenthesized keyword=value pairs where + // keyword is serverurl, proxyurl, or retrieve-ziplevel. (To be + // precise, parameterList may also be just a server URL; this + // string and any Connection serverURL other than the + // logicalServer are treated exactly the same, as either a single + // server if there's no parentheses or as a list of keyword=value + // pairs if there are parentheses). These are added after any + // servers or proxies passed to the Connection constructor. + static void setDefaultParams(const std::string& logicalServer, + const std::string& parameterList); + }; + +class Session + { + private: + unsigned long channel; + std::string *uri; + void *internal_data; + BLOB_TYPE last_field_type; + char have_saved_byte; + unsigned char saved_byte; + char first_row; + int num_records; + + public: + + explicit Session(Connection *myconnection); + + virtual ~Session(); + + // Get data for Requests + void getData(const std::vector& v_req); + + // Each Request generates a payload. Payload numbers started with 1. + // So, to get data for the first Request call setCurrentLoad(1) + void setCurrentLoad(int n); + + // Check error for this particular payload. + // If error happes for any payload that payload and all subsequent payloads (if any) are empty + int getCurrentLoadError() const; + // More detailed (hopefully) error explanation. + const char* getCurrentLoadErrorMessage() const; + + // Data fields extractors + // These methods change DS position to the next field + // Benchmarking had shown that inlining functions below does not improve performance + int getAnyData(AnyData* buf,int not_eor=1); + int getInt(); + long getLong(); + long long getLongLong(); + double getDouble(); + float getFloat(); + long long getDate(); + std::string *getString(); + std::string *getBlob(); + + void assignString(std::string *s); + + std::vector *getRawAsArrayUChar(); + std::vector *getRawAsArrayInt(); + std::vector *getRawAsArrayFloat(); + std::vector *getRawAsArrayDouble(); + std::vector *getRawAsArrayLong(); + + // Current pyload meta info + unsigned int getRecNum(); + unsigned int getNumberOfRecords(); + unsigned int getRSBinarySize(); + unsigned int getRSBinaryPos(); + BLOB_TYPE lastFieldType(){return last_field_type;} // Original type of the last extracted field + BLOB_TYPE nextFieldType(); // Next field type. THIS METHOD DOES NOT CHANGE DS POSITION !!! + inline int isEOR(){return (nextFieldType()==BLOB_TYPE_EOR);} // End Of Record. THIS METHOD DOES NOT CHANGE DS POSITION !!! + inline int isEOF(){return (getRSBinarySize()==getRSBinaryPos());} // End Of File + int next(); + void clean(); // for use from AnyData::clean() + }; + +// DataSource is a combined Session & Connection, for backward compatibility +class DataSource: public Connection, public Session + { + public: + + explicit DataSource(const std::string& server_url="",const std::string* proxy_url=NULL); + + // This constructor allows initialization with multiple server/proxies. + explicit DataSource(const std::list& serverUrlList, const std::list& proxyUrlList); + + virtual ~DataSource(); + }; + +} // namespace frontier + + +#endif //__HEADER_H_FRONTIER_FRONTIER_CPP_H_ + diff --git a/include/frontier_client/frontier.h b/include/frontier_client/frontier.h new file mode 100644 index 0000000..9648e8f --- /dev/null +++ b/include/frontier_client/frontier.h @@ -0,0 +1,107 @@ +/* + * frontier client C API header + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#ifndef __HEADER_H_FRONTIER_H +#define __HEADER_H_FRONTIER_H + +#include +#include "frontier_config.h" +#include "frontier_log.h" +#include "frontier_error.h" + +#define FRONTIER_MAX_PAYLOADNUM 32 + +typedef unsigned long FrontierChannel; +typedef void FrontierRSBlob; + +/*frontierRSBlob_get is deprecated, use frontierRSBlob_open instead*/ +FrontierRSBlob *frontierRSBlob_get(FrontierChannel u_channel,int n,int *ec); +FrontierRSBlob *frontierRSBlob_open(FrontierChannel u_channel,FrontierRSBlob *oldrs,int n,int *ec); +void frontierRSBlob_close(FrontierRSBlob *rs,int *ec); +void frontierRSBlob_rsctl(FrontierRSBlob *rs,int ctl_fn,void *data,int size,int *ec); +void frontierRSBlob_start(FrontierRSBlob *rs,int *ec); +char frontierRSBlob_getByte(FrontierRSBlob *rs,int *ec); +char frontierRSBlob_checkByte(FrontierRSBlob *rs,int *ec); // Returns next byte but do not change RS pointer +int frontierRSBlob_getInt(FrontierRSBlob *rs,int *ec); +long long frontierRSBlob_getLong(FrontierRSBlob *rs,int *ec); +double frontierRSBlob_getDouble(FrontierRSBlob *rs,int *ec); +float frontierRSBlob_getFloat(FrontierRSBlob *rs,int *ec); +char *frontierRSBlob_getByteArray(FrontierRSBlob *rs,unsigned int len,int *ec); +unsigned int frontierRSBlob_getRecNum(FrontierRSBlob *rs); +unsigned int frontierRSBlob_getPos(FrontierRSBlob *rs); +unsigned int frontierRSBlob_getSize(FrontierRSBlob *rs); +int frontierRSBlob_payload_error(FrontierRSBlob *rs); +const char* frontierRSBlob_payload_msg(FrontierRSBlob *rs); + +int frontier_initdebug(void *(*f_mem_alloc)(size_t size),void (*f_mem_free)(void *ptr), + const char *logfile, const char *loglevel); +int frontier_init(void *(*f_mem_alloc)(size_t size),void (*f_mem_free)(void *ptr)); + +FrontierChannel frontier_createChannel(const char *srv,const char *proxy,int *ec); +FrontierChannel frontier_createChannel2(FrontierConfig* config, int *ec); +void frontier_closeChannel(FrontierChannel chn); +void frontier_setTimeToLive(FrontierChannel u_channel,int ttl); // 1=short, 2=long, 3=forever +void frontier_setReload(FrontierChannel u_channel,int reload); // deprecated: 0=long ttl, !0=short ttl +int frontier_getRawData(FrontierChannel chn,const char *uri); +int frontier_postRawData(FrontierChannel chn,const char *uri,const char *body); +int frontier_getRetrieveZipLevel(FrontierChannel chn); + +int frontier_n2h_i32(const void* p); +float frontier_n2h_f32(const void* p); +long long frontier_n2h_i64(const void* p); +double frontier_n2h_d64(const void* p); + +size_t frontier_md5_get_ctx_size(); +void frontier_md5_init(void *ctx); +void frontier_md5_update(void *ctx,const unsigned char *data,unsigned int len); +void frontier_md5_final(void *ctx,unsigned char *out); + +char *frontier_str_ncopy(const char *str, size_t len); +char *frontier_str_copy(const char *str); + +void *frontier_malloc(size_t size); +void frontier_free(void *ptr); + +// GZip and base64URL encode +int fn_gzip_str2urlenc(const char *str,int size,char **out); + +// NOTE: The contents of this struct may not change or binary compatibility +// will break. +struct s_FrontierStatisticsNum + { + unsigned int min; + unsigned int max; + unsigned int avg; + }; +typedef struct s_FrontierStatisticsNum FrontierStatisticsNum; + +// NOTE: for binary compatibility, variables should never be removed +// from this structure, and new ones should only be added to the end. +struct s_FrontierStatistics + { + int num_queries; + int num_errors; // num with no response bytes; not included in other stats + FrontierStatisticsNum bytes_per_query; + FrontierStatisticsNum msecs_per_query; + }; +typedef struct s_FrontierStatistics FrontierStatistics; +void frontier_statistics_start(); +// return is status as defined in frontier_error.h +int frontier_statistics_get_bytes(FrontierStatistics *stats,int maxbytes); +#define frontier_statistics_get(STATS) frontier_statistics_get_bytes(STATS, sizeof(*(STATS))) +void frontier_statistics_stop(); + +#endif /*__HEADER_H_FRONTIER_H*/ + diff --git a/include/frontier_client/frontier_config.h b/include/frontier_client/frontier_config.h new file mode 100644 index 0000000..1731b92 --- /dev/null +++ b/include/frontier_client/frontier_config.h @@ -0,0 +1,112 @@ +/* + * frontier client config header + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#ifndef FRONTIER_CONFIG_H +#define FRONTIER_CONFIG_H + +#define FRONTIER_MAX_SERVERN 16 // Max number of servers in FRONTIER_SERVER env. variable +#define FRONTIER_MAX_PROXYN 24 // Max number of proxies in FRONTIER_PROXY env. variable +#define FRONTIER_MAX_PROXYCONFIGN 8 // Max number of proxyconfigurl + +#define FRONTIER_MAX_EPROTOAGE (5*60) // Max number of seconds to cache + // protocol errors +#define FRONTIER_DEFAULT_CLIENTCACHEMAXRESULTSIZE 0 + +#define FRONTIER_ENV_LOGICALSERVER "FRONTIER_LOGICALSERVER" +#define FRONTIER_ENV_PHYSICALSERVERS "FRONTIER_PHYSICALSERVERS" +#define FRONTIER_ENV_SERVER "FRONTIER_SERVER" +#define FRONTIER_ENV_PROXY "FRONTIER_PROXY" +#define FRONTIER_ENV_PROXYCONFIGS "FRONTIER_PROXYCONFIGS" +#define FRONTIER_ENV_RETRIEVEZIPLEVEL "FRONTIER_RETRIEVEZIPLEVEL" +#define FRONTIER_ENV_CONNECTTIMEOUTSECS "FRONTIER_CONNECTTIMEOUTSECS" +#define FRONTIER_ENV_READTIMEOUTSECS "FRONTIER_READTIMEOUTSECS" +#define FRONTIER_ENV_WRITETIMEOUTSECS "FRONTIER_WRITETIMEOUTSECS" +#define FRONTIER_ENV_FORCERELOAD "FRONTIER_FORCERELOAD" +#define FRONTIER_ENV_FRESHKEY "FRONTIER_FRESHKEY" + +struct s_FrontierConfig + { + char *server[FRONTIER_MAX_SERVERN]; + char *proxy[FRONTIER_MAX_PROXYN]; + char *proxyconfig[FRONTIER_MAX_PROXYCONFIGN]; + int server_num; + int proxy_num; + int proxyconfig_num; + int server_cur; + int proxy_cur; + int proxyconfig_cur; + int servers_balanced; + int proxies_balanced; + int num_backupproxies; + int connect_timeout_secs; + int read_timeout_secs; + int write_timeout_secs; + int max_age_secs; + char *force_reload; + char *freshkey; + int retrieve_zip_level; + int secured; + char *capath; + int client_cache_max_result_size; + int failover_to_server; + int prefer_ip_family; + }; +typedef struct s_FrontierConfig FrontierConfig; +FrontierConfig *frontierConfig_get(const char *server_url,const char *proxy_url,int *errorCode); +const char *frontierConfig_getServerUrl(FrontierConfig *cfg); +const char *frontierConfig_getProxyUrl(FrontierConfig *cfg); +int frontierConfig_nextServer(FrontierConfig *cfg); +int frontierConfig_nextProxy(FrontierConfig *cfg); +void frontierConfig_delete(FrontierConfig *cfg); +int frontierConfig_addServer(FrontierConfig *cfg, const char* server_url); +int frontierConfig_addProxy(FrontierConfig *cfg, const char* proxy_url, int backup); +int frontierConfig_addProxyConfig(FrontierConfig *cfg, const char* proxyconfig_url); +int frontierConfig_doProxyConfig(FrontierConfig *cfg); +void frontierConfig_setBalancedProxies(FrontierConfig *cfg); +int frontierConfig_getNumBalancedProxies(FrontierConfig *cfg); +void frontierConfig_setBalancedServers(FrontierConfig *cfg); +int frontierConfig_getBalancedServers(FrontierConfig *cfg); +void frontierConfig_setConnectTimeoutSecs(FrontierConfig *cfg,int secs); +int frontierConfig_getConnectTimeoutSecs(FrontierConfig *cfg); +void frontierConfig_setReadTimeoutSecs(FrontierConfig *cfg,int secs); +int frontierConfig_getReadTimeoutSecs(FrontierConfig *cfg); +void frontierConfig_setWriteTimeoutSecs(FrontierConfig *cfg,int secs); +int frontierConfig_getWriteTimeoutSecs(FrontierConfig *cfg); +void frontierConfig_setMaxAgeSecs(FrontierConfig *cfg,int secs); +int frontierConfig_getMaxAgeSecs(FrontierConfig *cfg); +void frontierConfig_setPreferIpFamily(FrontierConfig *cfg,int ipfamily); +int frontierConfig_getPreferIpFamily(FrontierConfig *cfg); +void frontierConfig_setForceReload(FrontierConfig *cfg,char *forcereload); +const char *frontierConfig_getForceReload(FrontierConfig *cfg); +void frontierConfig_setFreshkey(FrontierConfig *cfg,char *freshkey); +const char *frontierConfig_getFreshkey(FrontierConfig *cfg); +void frontierConfig_setRetrieveZipLevel(FrontierConfig *cfg,int level); +int frontierConfig_getRetrieveZipLevel(FrontierConfig *cfg); +void frontierConfig_setDefaultRetrieveZipLevel(int level); +int frontierConfig_getDefaultRetrieveZipLevel(); +void frontierConfig_setDefaultLogicalServer(const char *logical_server); +char *frontierConfig_getDefaultLogicalServer(); +void frontierConfig_setDefaultPhysicalServers(const char *physical_servers); +char *frontierConfig_getDefaultPhysicalServers(); +void frontierConfig_setSecured(FrontierConfig *cfg,int secured); +int frontierConfig_getSecured(FrontierConfig *cfg); +void frontierConfig_setCAPath(FrontierConfig *cfg,char *capath); +char *frontierConfig_getCAPath(FrontierConfig *cfg); +void frontierConfig_setClientCacheMaxResultSize(FrontierConfig *cfg,int size); +int frontierConfig_getClientCacheMaxResultSize(FrontierConfig *cfg); +void frontierConfig_setFailoverToServer(FrontierConfig *cfg,int notno); +int frontierConfig_getFailoverToServer(FrontierConfig *cfg); + +#endif /* FRONTIER_CONFIG_H */ + + diff --git a/include/frontier_client/frontier_error.h b/include/frontier_client/frontier_error.h new file mode 100644 index 0000000..50081d8 --- /dev/null +++ b/include/frontier_client/frontier_error.h @@ -0,0 +1,41 @@ +/* + * frontier client error API header + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#ifndef FRONTIER_ERROR_H +#define FRONTIER_ERROR_H + +#include "frontier_log.h" + +#define FRONTIER_OK 0 +#define FRONTIER_EIARG -1 /*Invalid argument passed*/ +#define FRONTIER_EMEM -2 /*mem_alloc failed*/ +#define FRONTIER_ECFG -3 /*config error*/ +#define FRONTIER_ESYS -4 /*system error*/ +#define FRONTIER_EUNKNOWN -5 /*unknown error*/ +#define FRONTIER_ENETWORK -6 /*error while communicating over network*/ +#define FRONTIER_EPROTO -7 /*protocol level error (e.g. wrong response)*/ +#define FRONTIER_ESERVER -8 /*server error (may be cached for short time)*/ +#define FRONTIER_ECONNECT -9 /*socket connect error*/ + +void frontier_vsetErrorMsg(const char *file, int line,const char *fmt,va_list ap); +void frontier_setErrorMsg(const char *file, int line,const char *fmt,...); +const char *frontier_get_err_desc(int err_num); +const char *frontier_getErrorMsg(); +void frontier_turnErrorsIntoDebugs(int value); + +#define FRONTIER_MSG(e) do{frontier_setErrorMsg(__FILE__,__LINE__,"error %d: %s",(e),frontier_get_err_desc(e));}while(0) + +#endif /* FRONTIER_ERROR_H */ + diff --git a/include/frontier_client/frontier_log.h b/include/frontier_client/frontier_log.h new file mode 100644 index 0000000..e7066cf --- /dev/null +++ b/include/frontier_client/frontier_log.h @@ -0,0 +1,33 @@ +/* + * frontier client logging header + * + * Author: Sinisa Veseli + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#ifndef FRONTIER_LOG_H +#define FRONTIER_LOG_H + +#include + +#define FRONTIER_LOGLEVEL_DEBUG 0 +#define FRONTIER_LOGLEVEL_WARNING 1 +#define FRONTIER_LOGLEVEL_INFO FRONTIER_LOGLEVEL_WARNING +#define FRONTIER_LOGLEVEL_ERROR 2 +#define FRONTIER_LOGLEVEL_NOLOG 3 + +int frontier_log_init(); +void frontier_vlog(int level,const char *file,int line,const char *fmt,va_list ap); +void frontier_log(int level,const char *file,int line,const char *fmt,...); +void frontier_log_close(); + +#endif /* FRONTIER_LOG_H */ + diff --git a/main.c b/main.c new file mode 100644 index 0000000..00e6f1f --- /dev/null +++ b/main.c @@ -0,0 +1,97 @@ +/* + * frontier client main for a standalone program + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ +#include +#include +#include +#include + + +#define CHECK(){if(ec!=FRONTIER_OK){printf("Error %d\n",ec); exit(1);}} + +static int mem_count=0; + +static void *my_malloc(size_t size) + { + ++mem_count; + printf("malloc, count %d\n",mem_count); + return malloc(size); + } + +static void my_free(void *ptr) + { + --mem_count; + printf("free, count %d\n",mem_count); + free(ptr); + } + + +int do_main(int argc, char **argv); + +int main(int argc, char **argv) + { + while(1) + { + do_main(argc,argv); + } + } + +int do_main(int argc, char **argv) + { + int ret; + FrontierChannel chnl; + FrontierRespStat stat; + FrontierRSBlob *rs; + int i; + int ec; + + if(argc!=2) + { + printf("Usage: %s server_url\n",argv[0]); + exit(1); + } + + ret=frontier_init(my_malloc,my_free); + if(ret) + { + printf("Error %d\n",ret); + exit(1); + } + + chnl=frontier_createChannel(argv[1],NULL,&ec); + CHECK() + + ret=frontier_getRawData(chnl,"Frontier?type=SvxBeamPosition:1&encoding=BLOB&cid=316011&type=CALTrigWeights:1&encoding=BLOB&cid=14319"); + if(ret) + { + printf("Error %d\n",ret); + exit(1); + } + + rs=frontierRSBlob_get(chnl,1,&ec); + CHECK() + for(i=0;i<100;i++) + { + int v=frontierRSBlob_getInt(rs,&ec); + CHECK() + printf("%d %d\n",i,v); + } + frontierRSBlob_close(rs,&ec); + + frontier_closeChannel(chnl); + + return(0); + } + + diff --git a/maincc.cc b/maincc.cc new file mode 100644 index 0000000..6127e34 --- /dev/null +++ b/maincc.cc @@ -0,0 +1,276 @@ +/* + * frontier client test C++ main program + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ +#include + +#include +#include + +class SvxBeamPosition + { + public: + long long CID; + long long CHANNELID; + double BEAMX; + double BEAMY; + double SLOPEX; + double SLOPEY; + double WIDTHX; + double WIDTHY; + double WIDTHXY; + double BEAMZ; + double WIDTHZ; + double BEAMXRATE; + double BEAMYRATE; + double SLOPEXRATE; + double SLOPEYRATE; + double WIDTHXRATE; + double WIDTHYRATE; + double WIDTHXYRATE; + double BEAMZRATE; + double WIDTHZRATE; + double MINIMZFIT; + double WIDTHZFIT; + double BSTARZFIT; + double ZZEROZFIT; + double MINIMZFITRATE; + double WIDTHZFITRATE; + double BSTARZFITRATE; + double ZZEROZFITRATE; + double EMITTXFIT; + double BSTARXFIT; + double ZZEROXFIT; + double EMITTYFIT; + double BSTARYFIT; + double ZZEROYFIT; + double EMITTXRATE; + double BSTARXRATE; + double ZZEROXRATE; + double EMITTYRATE; + double BSTARYRATE; + double ZZEROYRATE; + double FITCOV00; + double FITCOV01; + double FITCOV02; + double FITCOV03; + double FITCOV11; + double FITCOV12; + double FITCOV13; + double FITCOV22; + double FITCOV23; + double FITCOV33; + double STATISTICS0; + double STATISTICS1; + long long FLAG0; + long long FLAG1; + double SPARE0; + double SPARE1; + double SPARE2; + double SPARE3; + + SvxBeamPosition(frontier::DataSource& ds) + { + //CID=ds.getLongLong(); + CHANNELID=ds.getInt(); + BEAMX=ds.getDouble(); + BEAMY=ds.getDouble(); + SLOPEX=ds.getDouble(); + SLOPEY=ds.getDouble(); + WIDTHX=ds.getDouble(); + WIDTHY=ds.getDouble(); + WIDTHXY=ds.getDouble(); + BEAMZ=ds.getDouble(); + WIDTHZ=ds.getDouble(); + BEAMXRATE=ds.getDouble(); + BEAMYRATE=ds.getDouble(); + SLOPEXRATE=ds.getDouble(); + SLOPEYRATE=ds.getDouble(); + WIDTHXRATE=ds.getDouble(); + WIDTHYRATE=ds.getDouble(); + WIDTHXYRATE=ds.getDouble(); + BEAMZRATE=ds.getDouble(); + WIDTHZRATE=ds.getDouble(); + MINIMZFIT=ds.getDouble(); + WIDTHZFIT=ds.getDouble(); + BSTARZFIT=ds.getDouble(); + ZZEROZFIT=ds.getDouble(); + MINIMZFITRATE=ds.getDouble(); + WIDTHZFITRATE=ds.getDouble(); + BSTARZFITRATE=ds.getDouble(); + ZZEROZFITRATE=ds.getDouble(); + EMITTXFIT=ds.getDouble(); + BSTARXFIT=ds.getDouble(); + ZZEROXFIT=ds.getDouble(); + EMITTYFIT=ds.getDouble(); + BSTARYFIT=ds.getDouble(); + ZZEROYFIT=ds.getDouble(); + EMITTXRATE=ds.getDouble(); + BSTARXRATE=ds.getDouble(); + ZZEROXRATE=ds.getDouble(); + EMITTYRATE=ds.getDouble(); + BSTARYRATE=ds.getDouble(); + ZZEROYRATE=ds.getDouble(); + FITCOV00=ds.getDouble(); + FITCOV01=ds.getDouble(); + FITCOV02=ds.getDouble(); + FITCOV03=ds.getDouble(); + FITCOV11=ds.getDouble(); + FITCOV12=ds.getDouble(); + FITCOV13=ds.getDouble(); + FITCOV22=ds.getDouble(); + FITCOV23=ds.getDouble(); + FITCOV33=ds.getDouble(); + STATISTICS0=ds.getDouble(); + STATISTICS1=ds.getDouble(); + FLAG0=ds.getInt(); + FLAG1=ds.getInt(); + SPARE0=ds.getDouble(); + SPARE1=ds.getDouble(); + SPARE2=ds.getDouble(); + SPARE3=ds.getDouble(); + } + }; + + +class CalTrigWeights + { + public: + long long CID; + long long ID; + double TRIGSCL; + std::vector *ET_WEIGHT_CENT; + std::vector *ET_WEIGHT_WALL; + std::vector *ET_WEIGHT_PLUG; + std::vector *ET_FACTOR_CENT; + std::vector *ET_FACTOR_WALL; + std::vector *ET_FACTOR_PLUG; + + CalTrigWeights(frontier::DataSource& ds) + { + //CID=ds.getLongLong(); + ID=ds.getInt(); + TRIGSCL=ds.getDouble(); + ET_WEIGHT_CENT=ds.getRawAsArrayFloat(); + ET_WEIGHT_WALL=ds.getRawAsArrayFloat(); + ET_WEIGHT_PLUG=ds.getRawAsArrayFloat(); + ET_FACTOR_CENT=ds.getRawAsArrayInt(); + ET_FACTOR_WALL=ds.getRawAsArrayInt(); + ET_FACTOR_PLUG=ds.getRawAsArrayInt(); + } + + virtual ~CalTrigWeights() + { + delete ET_WEIGHT_CENT; + delete ET_WEIGHT_WALL; + delete ET_WEIGHT_PLUG; + delete ET_FACTOR_CENT; + delete ET_FACTOR_WALL; + delete ET_FACTOR_PLUG; + } + }; + + +int do_main(int argc, char **argv); + + +int main(int argc, char **argv) + { + while(1) + { + do_main(argc,argv); + break; + } + } + + +int do_main(int argc, char **argv) + { + std::string server_url=""; + std::string *proxy_url=NULL; + +#ifdef FNTR_USE_EXCEPTIONS + try + { +#endif //FNTR_USE_EXCEPTIONS + frontier::init(); + + if(argc>1) server_url=argv[1]; + if(argc>2) proxy_url=new std::string(argv[2]); + + frontier::DataSource ds(server_url,proxy_url); + if(proxy_url){delete proxy_url; proxy_url=NULL;} + //frontier::CDFDataSource ds; + + ds.setReload(1); + + frontier::Request req1("SvxBeamPosition:1",frontier::BLOB); + req1.addKey("cid","316011"); + frontier::Request req2("CALTrigWeights:1",frontier::BLOB); + req2.addKey("cid","14319"); + + std::vector vrq; + vrq.insert(vrq.end(),&req1); + vrq.insert(vrq.end(),&req2); + ds.getData(vrq); + + ds.setCurrentLoad(1); + + int nrec=ds.getRecNum(); + std::vector v_sbp(nrec); + for(int i=0;iCHANNELID<<','<BEAMX<<'\n'; + } + + // Do some usefull things here ... + + // Clean + for(int i=0;i v_ctw(nrec); + for(int i=0;iID<ET_WEIGHT_CENT->operator[](i)<<'\n'; + } + + // Clean + for(int i=0;i + +#include +#include +#include +#include + + +long long print_time(const char *msg) + { + struct timeval tv; + long long res; + + int ret=gettimeofday(&tv,NULL); + if(ret) printf("Can not get current time :-(\n"); + + res=((long long)(tv.tv_sec))*1000+((long long)(tv.tv_usec))/1000; + printf("%s %lld\n",msg,res); + + return res; + } + + +static void usage(char **argv) + { + printf("Usage: %s [-r] [-n count] [-s server] -o object:v[:m] key value [{key value}] [{-o object:v[:m] key value [{key value}] }]\n",argv[0]); + printf("\t -r - refresh cache on each request\n"); + printf("\t -n count - repeat 'count' times\n"); + printf("\t -s server - use this 'server' instead one from environment variables\n"); + printf("\nFrontier Client test program, see http://lynx.fnal.gov/ntier-wiki/ for details\n"); + exit(1); + } + + +int do_main(frontier::DataSource *ds,int obj1_ind,int argc, char **argv) + { + long long t1,t2; + std::vector vrq; + char **arg_p=argv+(obj1_ind+1); // Skipping first "-o" + + while(*arg_p) + { + for(int i=0;i<3;i++) + { + char *p=arg_p[i]; + if(!p) usage(argv); + if(strncmp(p,"-o",2)==0) usage(argv); + } + frontier::Request* req=new frontier::Request(arg_p[0],frontier::BLOB,arg_p[1],arg_p[2]); + arg_p+=3; + while(*arg_p && strncmp(*arg_p,"-o",2)) + { + if(!(*(arg_p+1)) || strncmp(*(arg_p+1),"-o",2)==0) usage(argv); + req->addKey(*arg_p,*(arg_p+1)); + arg_p+=2; + } + if(*arg_p && strncmp(*arg_p,"-o",2)==0) ++arg_p; + else if(*arg_p && strncmp(*arg_p,"-o",2)!=0) usage(argv); + vrq.insert(vrq.end(),req); + } + t1=print_time("start: "); + ds->getData(vrq); + t2=print_time("finish: "); + + for(unsigned int i=1;i<=vrq.size();i++) // XXX + { + ds->setCurrentLoad(i); + int nrec=ds->getRecNum(); + printf("Payload %d records %d bsize %d\n",i,nrec,ds->getRSBinarySize()); + delete vrq[i-1]; + } + printf("Duration %lld ms\n",t2-t1); + + return 1; + } + + +int main(int argc, char **argv) + { + int arg_ind; + int refresh=0; + int repeat=1; + char *server=0; + int opt_num=0; + + for(arg_ind=1;arg_ind1) usage(argv); + if(arg_ind+1>=argc || *(argv[arg_ind+1])=='-') usage(argv); + opt_num=2; + repeat=atoi(argv[arg_ind+1]); + ++arg_ind; + continue; + } + if(strncmp(argv[arg_ind],"-s",2)==0) + { + if(opt_num>2) usage(argv); + if(arg_ind+1>=argc || *(argv[arg_ind+1])=='-') usage(argv); + opt_num=3; + server=argv[arg_ind+1]; + arg_ind+=2; // Last option before objects + break; + } + } + + if(arg_ind>=argc || strncmp(argv[arg_ind],"-o",2)) usage(argv); + +#ifdef FNTR_USE_EXCEPTIONS + try + { +#endif //FNTR_USE_EXCEPTIONS + frontier::init(); + frontier::DataSource *ds; + + if(server) + { + ds=new frontier::DataSource(server); + } + else + { + ds=new frontier::DataSource(); + } + if(refresh) + { + printf("Refresh the cache.\n"); + ds->setReload(1); + } + + for(int i=1;i<=repeat;i++) + { + printf("Cycle %d of %d... ",i,repeat); + do_main(ds,arg_ind,argc,argv); + } + delete ds; +#ifdef FNTR_USE_EXCEPTIONS + } + catch(std::exception& e) + { + std::cout << "Error: " << e.what() << "\n"; + exit(1); + } + catch(...) + { + std::cout << "Unknown exception\n"; + exit(1); + } +#endif //FNTR_USE_EXCEPTIONS + + } + diff --git a/memdata.c b/memdata.c new file mode 100644 index 0000000..0594ead --- /dev/null +++ b/memdata.c @@ -0,0 +1,325 @@ +/* + * frontier client base64 memory handler + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#include +#include +#include +#include +#include +#include "fn-internal.h" +#include "fn-zlib.h" +#include "zlib.h" + +/* This is the size of each buffer after base64 decoding is applied. + The buffers get linked together and as many as needed are allocated + because we don't know ahead of time how much space is going to be + needed. Choose a size large enough that the overhead time of + malloc becomes insignificant for large queries, but small enough + that there isn't a lot of wasted space. + Just in case we're using a malloc that likes to allocate on page + boundaries, choose a size that fits nicely into a multiple of pages + after the FrontierMemBuf structure at the beginning of the allocated + space, minus a little more for whatever overhead malloc might take. + */ +#define MEMBUF_SIZE (16*4096-sizeof(FrontierMemBuf)-128) + +// decode %XX in strings, in place +static void +urldecode(char *src) + { + char *dst,a,b; + dst=src; + while(*src) + { + if((*src=='%')&& + ((a = src[1])&&(b = src[2]))&& + (isxdigit(a) && isxdigit(b))) + { + if(a>='a') a-='a'-'A'; + if(a>='A') a-=('A'-10); + else a-='0'; + if(b>='a') b-='a'-'A'; + if(b>='A') b-=('A'-10); + else b-='0'; + *dst++=16*a+b; + src+=3; + } + else *dst++=*src++; + } + *dst = '\0'; + } + +FrontierMemData *frontierMemData_create(int zipped,int secured,const char *params1,const char *params2) + { + FrontierMemData *md; + FrontierMemBuf *mb; + + md=frontier_mem_alloc(sizeof(*md)); + if(!md) return md; + + mb=(FrontierMemBuf *)frontier_mem_alloc(sizeof(*mb)+MEMBUF_SIZE); + if(!mb) goto err; + mb->nextbuf=0; + mb->len=0; + md->firstbuf=mb; + md->lastbuf=mb; + md->error=FRONTIER_OK; + md->total=0; + md->secured=secured; + bzero(&md->b64context,sizeof(md->b64context)); + if(secured) + { + bzero(md->sha256,sizeof(md->sha256)); + if (!SHA256_Init(&md->sha256_ctx)) + goto err; + // include the URL starting at the last '/' in the secure checksum + if(params1) + { + // only look for '/' before the first '&', just in case + char *p=strchr(params1,'&'); + if(p) *p='\0'; + params1=strrchr(params1,'/'); // there is always a slash + if(p) *p='&'; + + p=0; + if(strchr(params1,'%')!=0) + { + // Need to convert % encodings in URL into what server will see + // before calculating checksum + p=frontier_str_copy(params1); + urldecode(p); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"decoded URL for checksum: %s",p); + params1=p; + } + (void)SHA256_Update(&md->sha256_ctx,(unsigned char *)params1,strlen(params1)); + if(p) frontier_mem_free(p); + } + if(params2) + (void)SHA256_Update(&md->sha256_ctx,(unsigned char *)params2,strlen(params2)); + } + else + { + bzero(md->md5,sizeof(md->md5)); + if (!MD5_Init(&md->md5_ctx)) + goto err; + } + md->zipped_total=0; + md->binzipped=zipped; + md->zipbuflen=0; + fn_gunzip_init(); + return md; +err: + frontier_mem_free(md); + if(!mb)frontier_mem_free(mb); + return 0; + } + + +void frontierMemData_delete(FrontierMemData *md) + { + FrontierMemBuf *mb,*nextmb; + + if(!md) return; + + mb=md->firstbuf; + while(mb!=0) + { + nextmb=mb->nextbuf; + frontier_mem_free(mb); + mb=nextmb; + } + + frontier_mem_free(md); + } + +static FrontierMemBuf *frontierMemData_allocbuffer(FrontierMemData *md) + { + FrontierMemBuf *mb; + mb=(FrontierMemBuf *)frontier_mem_alloc(sizeof(*mb)+MEMBUF_SIZE); + if(!mb) return mb; + mb->nextbuf=0; + mb->len=0; + md->lastbuf->nextbuf=mb; + md->lastbuf=mb; + return mb; + } + +// append new base64-encoded data: decode it, calculate the message digest +// of the decoded data, and unzip it if it was zipped. Put the result in +// membufs. If it is a final update, finalize the message digest & unzipping. +static int frontierMemData_append(FrontierMemData *md, + const unsigned char *buf,int size,int final) + { + FrontierMemBuf *mb=md->lastbuf; + unsigned char *p=((unsigned char *)mb)+sizeof(*mb)+mb->len; + int spaceleft=MEMBUF_SIZE-mb->len; + int sizeused,spaceused; + unsigned char *p2=0; + int size2,spaceleft2; + + if(md->error!=FRONTIER_OK) + { + // if called again after an error, just return previous error + return md->error; + } + + if(md->binzipped) + { + // base64-decode into intermediate buffer "zipbuf" + p2=p; + spaceleft2=spaceleft; + p=&md->zipbuf[md->zipbuflen]; + spaceleft=sizeof(md->zipbuf)-md->zipbuflen; + } + + while(1) + { + sizeused=size; + spaceused=spaceleft; + fn_base64_stream_ascii2bin(&md->b64context,buf,&size,p,&spaceleft); + sizeused-=size; + spaceused-=spaceleft; + buf+=sizeused; + if(md->binzipped) + { + md->zipbuflen+=spaceused; + md->zipped_total+=spaceused; + if((size==0)&&(!final)) + { + // all the incoming buffer was copied to the zipbuf + return FRONTIER_OK; + } + // else finished filling the zipbuf, update message digest + if(md->secured) + (void)SHA256_Update(&md->sha256_ctx,((unsigned char *)&md->zipbuf[0]),md->zipbuflen); + else + (void)MD5_Update(&md->md5_ctx,((unsigned char *)&md->zipbuf[0]),md->zipbuflen); + // and unzip the zipbuf + p=md->zipbuf; + size2=md->zipbuflen; + while(1) + { + // unzip as much of the zipbuf as there's room for in the membuf + int ret; + sizeused=size2; + spaceused=spaceleft2; + ret=fn_gunzip_update(p,&size2,p2,&spaceleft2,final); + switch(ret) + { + case Z_OK: + break; + case Z_BUF_ERROR: + // this is fine, it just means there was no space for progress + // so allocate another buffer and try again + break; + case Z_MEM_ERROR: + frontier_setErrorMsg(__FILE__,__LINE__,"unzip memory error"); + md->error=FRONTIER_EMEM; + return md->error; + case Z_DATA_ERROR: + frontier_setErrorMsg(__FILE__,__LINE__,"unzip data error"); + md->error=FRONTIER_EPROTO; + return md->error; + default: + frontier_setErrorMsg(__FILE__,__LINE__,"unzip unknown error"); + md->error=FRONTIER_EUNKNOWN; + return md->error; + } + sizeused-=size2; + spaceused-=spaceleft2; + p+=sizeused; + mb->len+=spaceused; + md->total+=spaceused; + if(size2==0) + { + // all the zipbuf was uncompressed + md->zipbuflen=0; + if(final) + { + if(ret==Z_OK) + break; + // else Z_BUF_ERROR, allocate another membuf and try again + } + else + { + p=md->zipbuf; + spaceleft=sizeof(md->zipbuf); + break; + } + } + // else spaceleft2 must be zero, allocate another membuf + mb=frontierMemData_allocbuffer(md); + if(!mb) + { + FRONTIER_MSG(FRONTIER_EMEM); + return FRONTIER_EMEM; + } + p2=((unsigned char *)mb)+sizeof(*mb); + spaceleft2=MEMBUF_SIZE; + } + } + else + { + mb->len+=spaceused; + md->total+=spaceused; + if((size==0)&&!final) + return FRONTIER_OK; + //else finished with this membuf, update message digest + if(md->secured) + (void)SHA256_Update(&md->sha256_ctx,((unsigned char *)mb)+sizeof(*mb),mb->len); + else + (void)MD5_Update(&md->md5_ctx,((unsigned char *)mb)+sizeof(*mb),mb->len); + if(!final) + { + // allocate another membuf + mb=frontierMemData_allocbuffer(md); + if(!mb) + { + md->error=FRONTIER_EMEM; + FRONTIER_MSG(md->error); + return md->error; + } + p=((unsigned char *)mb)+sizeof(*mb); + spaceleft=MEMBUF_SIZE; + } + } + if(final) + { + if(md->secured) + (void)SHA256_Final(md->sha256,&md->sha256_ctx); + else + (void)MD5_Final(md->md5,&md->md5_ctx); + return FRONTIER_OK; + } + } + } + +int frontierMemData_b64append(FrontierMemData *md,const char *buf,int size) + { + return frontierMemData_append(md,(const unsigned char *)buf,size,0); + } + +int frontierMemData_finalize(FrontierMemData *md) + { + return frontierMemData_append(md,0,0,1); + } + +unsigned char *frontierMemData_getDigest(FrontierMemData *md) + { + if(md->secured) + return(md->sha256); + else + return md->md5; + } diff --git a/pacparser-dlopen.c b/pacparser-dlopen.c new file mode 100644 index 0000000..cc2bee3 --- /dev/null +++ b/pacparser-dlopen.c @@ -0,0 +1,122 @@ +/* + * pacparser dlopen interface + * + * Author: Dave Dykstra + * + * $Id$ + * + * Copyright (c) 2013, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#include +#include "pacparser.h" +#include "frontier_client/frontier_error.h" + +static void *pp_dlhandle; +static int (*pp_init)(void); +static void (*pp_set_error_printer)(pacparser_error_printer); +static void (*pp_setmyip)(const char *); +static int (*pp_parse_pac_string)(const char *); +static char *(*pp_find_proxy)(const char *,const char *); +static void (*pp_cleanup)(void); + +static void *pp_funcps[] = + { + &pp_init, + &pp_set_error_printer, + &pp_setmyip, + &pp_parse_pac_string, + &pp_find_proxy, + &pp_cleanup, + }; +static const char *pp_names[]= + { + "pacparser_init", + "pacparser_set_error_printer", + "pacparser_setmyip", + "pacparser_parse_pac_string", + "pacparser_find_proxy", + "pacparser_cleanup", + }; + +int frontier_pacparser_init(void) + { + const char *error; + int i; + + /* add .1 on the end because if the API ever changes in an incompatible + * way they're supposed to change it to .2 */ + pp_dlhandle=dlopen("libpacparser.so.1",RTLD_LAZY); + if(!pp_dlhandle) + { + frontier_setErrorMsg(__FILE__,__LINE__, + "config error: cannot dlopen %s",dlerror()); + return FRONTIER_ECFG; + } + + dlerror(); /* Clear any existing error */ + + for(i=0;i<(sizeof(pp_names)/sizeof(pp_names[0]));i++) + { + *(void **)(pp_funcps[i])=dlsym(pp_dlhandle,pp_names[i]); + if((error=dlerror())!=0) + { + frontier_setErrorMsg(__FILE__,__LINE__, "config error: dlsym %s",error); + dlclose(pp_dlhandle); + pp_dlhandle=0; + return FRONTIER_ECFG; + } + } + + return FRONTIER_OK; + } + +int pacparser_init(void) + { + if(!pp_dlhandle) + return 0; + return (*pp_init)(); + } + +void pacparser_set_error_printer(pacparser_error_printer func) + { + if(!pp_dlhandle) + return; + (*pp_set_error_printer)(func); + return; + } + +void pacparser_setmyip(const char *ip) + { + if(!pp_dlhandle) + return; + (*pp_setmyip)(ip); + } + +int pacparser_parse_pac_string(const char *string) + { + if(!pp_dlhandle) + return 0; + return (*pp_parse_pac_string)(string); + } + +char *pacparser_find_proxy(const char *url,const char *host) + { + if(!pp_dlhandle) + return 0; + return (*pp_find_proxy)(url,host); + } + +void pacparser_cleanup(void) + { + if(!pp_dlhandle) + return; + (*pp_cleanup)(); + dlclose(pp_dlhandle); + pp_dlhandle=0; + } diff --git a/payload.c b/payload.c new file mode 100644 index 0000000..3047019 --- /dev/null +++ b/payload.c @@ -0,0 +1,228 @@ +/* + * frontier client payload handler + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ + +#include +#include "fn-internal.h" +#include +#include +#include +#include + + +FrontierPayload *frontierPayload_create(const char *encoding,int secured,const char *params1,const char *params2) + { + FrontierPayload *fpl; + + fpl=frontier_mem_alloc(sizeof(FrontierPayload)); + if(!fpl) + { + FRONTIER_MSG(FRONTIER_EMEM); + return fpl; + } + + fpl->id=-1; + if(encoding==NULL) + fpl->encoding=(void*)0; + else + fpl->encoding=frontier_str_copy(encoding); + fpl->md=frontierMemData_create((strstr(encoding,"zip")!=NULL),secured,params1,params2); + if(!fpl->md) + { + frontierPayload_delete(fpl); + FRONTIER_MSG(FRONTIER_EMEM); + return (void*)0; + } + + fpl->blob=(void*)0; + fpl->blob_size=0; + + bzero(fpl->srv_md5_str,sizeof(fpl->srv_md5_str)); + fpl->srv_sig_len=0; + fpl->srv_sig=0; + + fpl->error=0; + fpl->error_code=0; + fpl->error_msg=(void*)0; + + return fpl; + } + + +void frontierPayload_delete(FrontierPayload *fpl) + { + if(!fpl) return; + + if(fpl->blob) frontier_mem_free(fpl->blob); + + if(fpl->encoding) frontier_mem_free(fpl->encoding); + + if(fpl->error_msg) frontier_mem_free(fpl->error_msg); + + if(fpl->srv_sig) frontier_mem_free(fpl->srv_sig); + + if(fpl->md) frontierMemData_delete(fpl->md); + + frontier_mem_free(fpl); + } + + +void frontierPayload_append(FrontierPayload *fpl,const char *s,int len) + { + frontierMemData_b64append(fpl->md,s,len); + } + + + +int frontierPayload_finalize(FrontierPayload *fpl) + { + int i; + int zipped=0; + int zipped_size=0; + unsigned char *p; + FrontierMemBuf *mb; + + fpl->blob = 0; + fpl->error = FRONTIER_OK; + + if(fpl->error_code!=FRONTIER_OK) + { + frontier_setErrorMsg(__FILE__,__LINE__,"Server signalled payload error %d: %s",fpl->error_code,fpl->error_msg); + fpl->error=FRONTIER_ESERVER; + goto errcleanup; + } + + if(fpl->encoding==0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"Encoding not specified in response"); + fpl->error=FRONTIER_EPROTO; + goto errcleanup; + } + + if(strncmp(fpl->encoding,"BLOB",4)!=0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"Unrecognized encoding type: %s",fpl->encoding); + fpl->error=FRONTIER_EPROTO; + goto errcleanup; + } + + if(fpl->encoding[4]!='\0') + { + if (strncmp(&fpl->encoding[4],"zip",3)!=0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"Unrecognized BLOB encoding subtype: %s",&fpl->encoding[4]); + fpl->error=FRONTIER_EPROTO; + goto errcleanup; + } + zipped=1; + } + + // finalize the MemData. + fpl->error=frontierMemData_finalize(fpl->md); + if(fpl->error!=FRONTIER_OK) + goto errcleanup; + + bcopy(frontierMemData_getDigest(fpl->md),fpl->digest,sizeof(fpl->digest)); + + // put all the buffered pieces together into one + fpl->blob_size=fpl->md->total; + fpl->blob=(unsigned char*)frontier_mem_alloc(fpl->blob_size); + if(!fpl->blob) + { + // This is the most likely place in all of the frontier client + // to get an error allocating memory, so report the size + frontier_setErrorMsg(__FILE__,__LINE__,"cannot allocate memory of size %d", + fpl->blob_size); + fpl->error=FRONTIER_EMEM; + goto errcleanup; + } + mb=fpl->md->firstbuf; + p=fpl->blob; + while(mb!=0) + { + bcopy(((unsigned char *)mb)+sizeof(*mb),p,mb->len); + p+=mb->len; + mb=mb->nextbuf; + } + + zipped_size=fpl->md->zipped_total; + frontierMemData_delete(fpl->md); + fpl->md=0; + + if (zipped) + { + // "uncompressed" is more accurate now but leave "uncompressing" + // because frontierqueries tool looks for that keyword + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"uncompressing %d byte (full size %d) payload",zipped_size,fpl->full_size); + } + else + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"finalizing payload (full size %d)",fpl->blob_size); + } + + if(frontier_log_level>=FRONTIER_LOGLEVEL_DEBUG) + { + unsigned char dumpdata[80*5+1]; + int n=0; + unsigned char c; + for(i=0;(iblob_size)&&(n<(sizeof(dumpdata)-sizeof("%00")));i++) + { + c=fpl->blob[i]; + if(c==0x07) + { + /* 0x07 separates records */ + dumpdata[n++]='\n'; + } + else if(c<=26) + { + dumpdata[n++]='^'; + if(c==0) + dumpdata[n++]='@'; + else + dumpdata[n++]=c+'a'-1; + } + else if(isprint(c)) + dumpdata[n++]=c; + else + n+=sprintf((char *)&dumpdata[n],"%%%02x",c); + } + dumpdata[n]='\0'; + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"start of decoded response: %s",dumpdata); + } + + if (fpl->blob_size != fpl->full_size) + { + frontier_setErrorMsg(__FILE__,__LINE__,"BLOB decoded size %d did not match expected size %d",fpl->blob_size,fpl->full_size); + fpl->error=FRONTIER_EPROTO; + goto errcleanup; + } + + //printf("Blob size %d\n",fpl->blob_size); + + return FRONTIER_OK; + +errcleanup: + if(fpl->md) + { + frontierMemData_delete(fpl->md); + fpl->md=0; + } + if(fpl->blob) + { + frontier_mem_free(fpl->blob); + fpl->blob=0; + } + return fpl->error; + } + diff --git a/request_each.cc b/request_each.cc new file mode 100644 index 0000000..4f042cd --- /dev/null +++ b/request_each.cc @@ -0,0 +1,84 @@ +/* + * frontier client test C++ main program + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ +#include + +#include +#include +#include + + +int main(int argc, char **argv) + { + if(argc!=3) + { + std::cout<<"Usage: "< vrq; + vrq.insert(vrq.end(),&req); + ds.getData(vrq); + + ds.setCurrentLoad(1); + int nrec=ds.getRecNum(); + std::cout<<"Got "< v_cid; + + for(int i=0;i vrq1; + vrq1.insert(vrq1.end(),&req1); + vrq1.insert(vrq1.end(),&req2); + ds.getData(vrq1); + ds.setCurrentLoad(2); + std::cout<<" nrec "< +#include +#include +#include "fn-internal.h" +#include "expat.h" +#include +#include +#include + +static void XMLCALL +xml_cdata(void *userData,const XML_Char *s,int len) + { + FrontierResponse *fr=(FrontierResponse*)userData; + + frontierPayload_append(fr->payload[fr->payload_num-1],s,len); + + //printf("xml_cdata\n"); + } + + +static void XMLCALL +xml_startElement(void *userData,const char *name,const char **atts) + { + int i; + FrontierResponse *fr=(FrontierResponse*)userData; + + //printf("xml_start %s\n",name); + + if(strcmp(name,"keepalive")==0) + fr->keepalives++; + else if(fr->keepalives!=0) + { + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"received %d keepalives",fr->keepalives); + fr->keepalives=0; + } + + if(strcmp(name,"global_error")==0) + { + fr->error=FRONTIER_EPROTO; + frontier_setErrorMsg(__FILE__,__LINE__,"Server has signalled Global Error [%s]",atts[1]); + return; + } + + if(strcmp(name,"frontier")==0) + { + char buf[80]; + int i; + size_t l=0; + buf[0] = '\0'; + for(i=0;atts[i];i+=2) + { + l+=snprintf(&buf[l],sizeof(buf)-l," %s=%s",atts[i],atts[i+1]); + if (l>=sizeof(buf)) + break; + } + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"frontier server%s",buf); + return; + } + + if(strcmp(name,"payload")==0) + { + const char *encoding=NULL; + fr->payload_num++; + for(i=0;atts[i];i+=2) + { + if(strcmp(atts[i],"encoding")==0) + { + encoding=atts[i+1]; + continue; + } + } + fr->payload[fr->payload_num-1]=frontierPayload_create(encoding,(fr->srv_rsakey!=0),fr->params1,fr->params2); + fr->p_state=FNTR_WITHIN_PAYLOAD; + return; + } + + if(strcmp(name,"data")==0) + { + XML_SetCharacterDataHandler(fr->parser,xml_cdata); + return; + } + + if(strcmp(name,"quality")==0 && fr->p_state==FNTR_WITHIN_PAYLOAD) + { + for(i=0;atts[i];i+=2) + { + //printf("attr <%s><%s>\n",atts[i],atts[i+1]); + fflush(stdout); + if(strcmp(atts[i],"error")==0) + { + fr->payload[fr->payload_num-1]->error_code=atoi(atts[i+1]); + continue; + } + if(strcmp(atts[i],"message")==0) + { + fr->payload[fr->payload_num-1]->error_msg=frontier_str_copy(atts[i+1]); + continue; + } + if(strcmp(atts[i],"max_age")==0) + { + int max_age=atoi(atts[i+1]); + if((fr->max_age<=0)||(max_agemax_age)) + fr->max_age=max_age; + continue; + } + if(strcmp(atts[i],"records")==0) + { + //printf("Number of records: %d\n", atoi(atts[i+1])); + fr->payload[fr->payload_num-1]->nrec=atoi(atts[i+1]); + continue; + } + if(strcmp(atts[i],"full_size")==0) + { + fr->payload[fr->payload_num-1]->full_size=atoi(atts[i+1]); + continue; + } + if(strcmp(atts[i],"md5")==0) + { + bcopy(atts[i+1],fr->payload[fr->payload_num-1]->srv_md5_str,MD5_DIGEST_LENGTH*2); + fr->payload[fr->payload_num-1]->srv_md5_str[MD5_DIGEST_LENGTH*2]=0; + continue; + } + if(strcmp(atts[i],"sig")==0) + { + int len=strlen(atts[i+1]); + unsigned char *srv_sig=frontier_mem_alloc(len*3/4); + if(srv_sig) + { + fr->payload[fr->payload_num-1]->srv_sig_len= + fn_base64_ascii2bin((unsigned char *)atts[i+1],len,srv_sig,len*3/4); + fr->payload[fr->payload_num-1]->srv_sig=srv_sig; + } + } + } + } + } + + +static void XMLCALL +xml_endElement(void *userData,const char *name) + { + int ret; + + FrontierResponse *fr=(FrontierResponse*)userData; + + //printf("xml_end %s\n",name); + + if(strcmp(name,"data")==0) + { + XML_SetCharacterDataHandler(fr->parser,(void*)0); + return; + } + + if(strcmp(name,"payload")==0) + { + ret=frontierPayload_finalize(fr->payload[fr->payload_num-1]); + fr->p_state=0; + fr->error=ret; + } + } + + + +FrontierResponse *frontierResponse_create(int *ec,void *srv_rsakey,const char *params1,const char *params2) + { + FrontierResponse *fr; + int i; + + fr=frontier_mem_alloc(sizeof(FrontierResponse)); + if(!fr) + { + *ec=FRONTIER_EMEM; + FRONTIER_MSG(*ec); + return fr; + } + + fr->error=0; + fr->payload_num=0; + fr->error_payload_ind=-1; + fr->keepalives=0; + fr->max_age=-1; + fr->srv_rsakey=srv_rsakey; + fr->params1=params1; + fr->params2=params2; + + fr->parser=XML_ParserCreate(NULL); + if(!fr->parser) + { + frontier_mem_free(fr); + *ec=FRONTIER_EUNKNOWN; + frontier_setErrorMsg(__FILE__,__LINE__,"Can not create XML parser instance."); + return (void*)0; + } + XML_SetUserData(fr->parser,fr); + XML_SetElementHandler(fr->parser,xml_startElement,xml_endElement); + + fr->p_state=0; + + for(i=0;ipayload[i]=(void*)0; + } + + return fr; + } + + +void frontierResponse_delete(FrontierResponse *fr) + { + int i; + + if(!fr) return; + + if(fr->parser) + { + XML_SetUserData(fr->parser,(void*)0); + XML_ParserFree(fr->parser); + } + + for(i=0;ipayload_num;i++) + { + frontierPayload_delete(fr->payload[i]); + } + + frontier_mem_free(fr); + } + + +int FrontierResponse_append(FrontierResponse *fr,char *buf,int len) + { + if(XML_Parse(fr->parser,buf,len,0)==XML_STATUS_ERROR) + { + int xml_err=XML_GetErrorCode(fr->parser); + frontier_setErrorMsg(__FILE__,__LINE__,"XML parse error %d:%s at line %d",xml_err,XML_ErrorString(xml_err),XML_GetCurrentLineNumber(fr->parser)); + return FRONTIER_EPROTO; + } + + if(fr->error) + { + return fr->error; + } + + return FRONTIER_OK; + } + + +int frontierResponse_finalize(FrontierResponse *fr) + { + int i; + unsigned char *digest; + + if(XML_Parse(fr->parser,"",0,1)==XML_STATUS_ERROR) + { + int xml_err=XML_GetErrorCode(fr->parser); + frontier_setErrorMsg(__FILE__,__LINE__,"XML parse error %d:%s at line %d",xml_err,XML_ErrorString(xml_err),XML_GetCurrentLineNumber(fr->parser)); + return FRONTIER_EPROTO; + } + + XML_SetUserData(fr->parser,(void*)0); + XML_ParserFree(fr->parser); + fr->parser=(void*)0; + + for(i=0;ipayload_num;i++) + { + //printf("%d r:[%s] l:[%s]\n",i,fr->payload[i]->srv_md5_str); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Payload[%d] error %d error code %d",(i+1),fr->payload[i]->error,fr->payload[i]->error_code); + if(fr->payload[i]->error) + { + frontier_setErrorMsg(__FILE__,__LINE__,"Payload[%d] got error %d",(i+1),fr->payload[i]->error); + return fr->payload[i]->error; + } + if(fr->payload[i]->error_code) + { + frontier_setErrorMsg(__FILE__,__LINE__,"Payload[%d] has error code %d",(i+1),fr->payload[i]->error_code); + return FRONTIER_EPROTO; + } + + digest=fr->payload[i]->digest; + + if(fr->srv_rsakey!=0) + { + unsigned char *sha256_buf=0; + int ret=FRONTIER_OK; + int siglen=fr->payload[i]->srv_sig_len; + int declen; + + if(siglen==0) + { + frontier_setErrorMsg(__FILE__,__LINE__,"Payload[%d]: no signature found",i+1); + return FRONTIER_EPROTO; + } + + // If rsa decrypt fails, the output can be as big as the input. + // If it succeeds, the length should be SHA256_DIGEST_LENGTH. + // So malloc a buffer of the bigger size just in case + sha256_buf=frontier_mem_alloc(siglen); + if(!sha256_buf) {ret=FRONTIER_EMEM;FRONTIER_MSG(ret);return ret;}; + declen=RSA_public_decrypt(siglen,fr->payload[i]->srv_sig,sha256_buf, + (RSA *)fr->srv_rsakey,RSA_PKCS1_PADDING); + if(declen==-1) + { + SSL_load_error_strings(); + frontier_setErrorMsg(__FILE__,__LINE__,"Payload[%d]: error decrypting signature: %s",i+1,ERR_error_string(ERR_get_error(),0)); + ERR_free_strings(); + ret=FRONTIER_EPROTO; + goto endsigned; + } + if(declen!=SHA256_DIGEST_LENGTH) + { + frontier_setErrorMsg(__FILE__,__LINE__,"Payload[%d]: decrypted signature wrong length; expect %d, got %d",i+1,SHA256_DIGEST_LENGTH,declen); + ret=FRONTIER_EPROTO; + goto endsigned; + } + + if(memcmp(digest,sha256_buf,declen)==0) + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"Payload[%d] signature passed",i+1); + else + { + char digest_str[SHA256_DIGEST_LENGTH*2+1]; + char sha256_str[SHA256_DIGEST_LENGTH*2+1]; + for(i=0;ipayload[0]->srv_md5_str)]; + char *srv_md5_str; + + // this variable is set here to avoid a compiler bug; moving this to + // just before the strncmp below (or eliminating the variable) loses + // the upper bits & causes a seg fault, at least on gcc 4.1.2 + srv_md5_str=fr->payload[i]->srv_md5_str; + + bzero(md5_str,sizeof(md5_str)); + // convert the binary md5 characters into printable + for(i=0;i +#include +#include +#include +#include "fn-internal.h" +#include "fn-endian.h" + +#define TYPED_BLOB + +union u_Buf32 { int v; float f; char b[4]; }; +union u_Buf64 { long long v; double d; char b[8]; }; + +#if __BYTE_ORDER == __LITTLE_ENDIAN +static inline int n2h_i32(const unsigned char *p) + { + union u_Buf32 u; + u.b[3]=p[0]; + u.b[2]=p[1]; + u.b[1]=p[2]; + u.b[0]=p[3]; + return u.v; + } +static inline float n2h_f32(const unsigned char *p) + { + union u_Buf32 u; + u.b[3]=p[0]; + u.b[2]=p[1]; + u.b[1]=p[2]; + u.b[0]=p[3]; + return u.f; + } +static inline long long n2h_i64(const unsigned char *p) + { + union u_Buf64 u; + u.b[7]=p[0]; + u.b[6]=p[1]; + u.b[5]=p[2]; + u.b[4]=p[3]; + u.b[3]=p[4]; + u.b[2]=p[5]; + u.b[1]=p[6]; + u.b[0]=p[7]; + return u.v; + } +static inline double n2h_d64(const unsigned char *p) + { + union u_Buf64 u; + u.b[7]=p[0]; + u.b[6]=p[1]; + u.b[5]=p[2]; + u.b[4]=p[3]; + u.b[3]=p[4]; + u.b[2]=p[5]; + u.b[1]=p[6]; + u.b[0]=p[7]; + return u.d; + } +#else +#warning Big endian order +#include +static inline int n2h_i32(const unsigned char *p) + { + return *((int*)p); + } +static inline float n2h_f32(const unsigned char *p) + { + return *((float*)p); + } +static inline long long n2h_i64(const unsigned char *p) + { + return *((long long*)p); + } +static inline double n2h_d64(const unsigned char *p) + { + return *((double*)p); + } +#endif /*__BYTE_ORDER*/ + +int frontier_n2h_i32(const void* p){return n2h_i32(p);} +float frontier_n2h_f32(const void* p){return n2h_f32(p);} +long long frontier_n2h_i64(const void* p){return n2h_i64(p);} +double frontier_n2h_d64(const void* p){return n2h_d64(p);} + +/*this function is for backward compatibility*/ +FrontierRSBlob *frontierRSBlob_get(FrontierChannel u_channel,int n,int *ec) + { + return frontierRSBlob_open(u_channel,0,n,ec); + } + +FrontierRSBlob *frontierRSBlob_open(FrontierChannel u_channel,FrontierRSBlob *oldfrs,int n,int *ec) + { + Channel *chn=(Channel*)u_channel; + FrontierResponse *resp; + FrontierPayload *fp; + FrontierRSBlob *frs=(void*)0; + RSBlob *rs=(void*)0; + RSBlob *oldrs=(RSBlob *)oldfrs; + + if(oldrs&&(oldrs->resp)) + { + /*move the response from the old RSblob to here*/ + resp=oldrs->resp; + oldrs->resp=0; + } + else + { + /*move the response from the channel to here*/ + resp=chn->resp; + chn->resp=0; + } + + if(n>resp->payload_num) + { + *ec=FRONTIER_EIARG; + frontier_setErrorMsg(__FILE__,__LINE__,"no such payload - total %d, requested %d",resp->payload_num,n); + if(resp)frontierResponse_delete(resp); + return frs; + } + + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__,"parsing chan %d response %d payload %d",chn->seqnum,resp->seqnum,n); + + fp=resp->payload[n-1]; + + rs=frontier_mem_alloc(sizeof(RSBlob)); + frs=(FrontierRSBlob *)rs; + if(!frs) + { + *ec=FRONTIER_EMEM; + FRONTIER_MSG(*ec); + if(resp)frontierResponse_delete(resp); + return frs; + } + + rs->resp=resp; + rs->buf=fp->blob; + rs->size=fp->blob_size; + rs->pos=0; + rs->nrec=fp->nrec; + rs->respnum=resp->seqnum; // for debugging + + rs->payload_error=fp->error_code; + rs->payload_msg=fp->error_msg; + + *ec=FRONTIER_OK; + return frs; + } + + + +void frontierRSBlob_close(FrontierRSBlob *frs,int *ec) + { + RSBlob *rs=(RSBlob *)frs; + *ec=FRONTIER_OK; + + if(!rs) return; + if(rs->resp)frontierResponse_delete(rs->resp); + frontier_mem_free(rs); + } + + +void frontierRSBlob_rsctl(FrontierRSBlob *frs,int ctl_fn,void *data,int size,int *ec) + { + *ec=FRONTIER_OK; + } + + +void frontierRSBlob_start(FrontierRSBlob *frs,int *ec) + { + RSBlob* rs=(RSBlob *)frs; + *ec=FRONTIER_OK; + rs->pos=0; + } + + + +char frontierRSBlob_getByte(FrontierRSBlob *frs,int *ec) + { + RSBlob* rs=(RSBlob *)frs; + char ret; + + if(rs->pos>=rs->size) + { + *ec=FRONTIER_EIARG; + frontier_setErrorMsg(__FILE__,__LINE__,"resultset size (%d bytes) violation",rs->size); + return (char)-1; + } + ret=rs->buf[rs->pos]; + rs->pos++; + *ec=FRONTIER_OK; + return ret; + } + + +char frontierRSBlob_checkByte(FrontierRSBlob *frs,int *ec) + { + RSBlob* rs=(RSBlob *)frs; + char ret; + + if(rs->pos>=rs->size) + { + *ec=FRONTIER_EIARG; + frontier_setErrorMsg(__FILE__,__LINE__,"resultset size (%d bytes) violation",rs->size); + return (char)-1; + } + ret=rs->buf[rs->pos]; + *ec=FRONTIER_OK; + return ret; + } + + +char *frontierRSBlob_getByteArray(FrontierRSBlob *frs,unsigned int len,int *ec) + { + RSBlob* rs=(RSBlob *)frs; + const unsigned char *buf; + + if(rs->pos>=rs->size-(len-1)) + { + *ec=FRONTIER_EIARG; + frontier_setErrorMsg(__FILE__,__LINE__,"resultset size (%d bytes) violation",rs->size); + return 0; + } + buf=rs->buf+rs->pos; + rs->pos+=len; + *ec=FRONTIER_OK; + return (char *)buf; + } + + +int frontierRSBlob_getInt(FrontierRSBlob *frs,int *ec) + { + RSBlob* rs=(RSBlob *)frs; + int ret; + + if(rs->pos>=rs->size-3) + { + *ec=FRONTIER_EIARG; + frontier_setErrorMsg(__FILE__,__LINE__,"resultset size (%d bytes) violation",rs->size); + return -1; + } + ret=n2h_i32(rs->buf+rs->pos); + rs->pos+=4; + *ec=FRONTIER_OK; + return ret; + } + + +long long frontierRSBlob_getLong(FrontierRSBlob *frs,int *ec) + { + RSBlob* rs=(RSBlob *)frs; + long long ret; + + if(rs->pos>=rs->size-7) + { + *ec=FRONTIER_EIARG; + frontier_setErrorMsg(__FILE__,__LINE__,"resultset size (%d bytes) violation",rs->size); + return -1; + } + ret=n2h_i64(rs->buf+rs->pos); + rs->pos+=8; + *ec=FRONTIER_OK; + return ret; + } + + +double frontierRSBlob_getDouble(FrontierRSBlob *frs,int *ec) + { + RSBlob* rs=(RSBlob *)frs; + double ret; + + if(rs->pos>=rs->size-7) + { + *ec=FRONTIER_EIARG; + frontier_setErrorMsg(__FILE__,__LINE__,"resultset size (%d bytes) violation",rs->size); + return -1; + } + ret=n2h_d64(rs->buf+rs->pos); + rs->pos+=8; + *ec=FRONTIER_OK; + return ret; + } + + +float frontierRSBlob_getFloat(FrontierRSBlob *frs,int *ec) + { + RSBlob* rs=(RSBlob *)frs; + float ret; + + if(rs->pos>=rs->size-3) + { + *ec=FRONTIER_EIARG; + frontier_setErrorMsg(__FILE__,__LINE__,"resultset size (%d bytes) violation",rs->size); + return -1; + } + ret=n2h_f32(rs->buf+rs->pos); + rs->pos+=4; + *ec=FRONTIER_OK; + return ret; + } + + +unsigned int frontierRSBlob_getRecNum(FrontierRSBlob *frs) + { + RSBlob* rs=(RSBlob *)frs; + return rs->nrec; + } + +unsigned int frontierRSBlob_getPos(FrontierRSBlob *frs) + { + RSBlob* rs=(RSBlob *)frs; + return rs->pos; + } + +unsigned int frontierRSBlob_getSize(FrontierRSBlob *frs) + { + RSBlob* rs=(RSBlob *)frs; + return rs->size; + } + +int frontierRSBlob_payload_error(FrontierRSBlob *frs) + { + RSBlob* rs=(RSBlob *)frs; + return rs->payload_error; + } + +const char *frontierRSBlob_payload_msg(FrontierRSBlob *frs) + { + RSBlob* rs=(RSBlob *)frs; + return rs->payload_msg; + } + + diff --git a/statistics.c b/statistics.c new file mode 100644 index 0000000..766d220 --- /dev/null +++ b/statistics.c @@ -0,0 +1,145 @@ +/* + * frontier client statistics API implementation + * + * Author: Dave Dykstra + * + * Copyright (c) 2020, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt. + * + */ + +#include +#include +#include +#include +#include +#include "fn-internal.h" + +static FrontierStatistics *statistics; +int started_by_debug=0; + +void frontier_statistics_start() + { + (void) frontier_init(NULL, NULL); + + if(statistics!=NULL) + return; + + started_by_debug=0; + + statistics=frontier_mem_alloc(sizeof(FrontierStatistics)); + if(statistics==NULL) + { + // out of memory already, so we'll skip taking stats + return; + } + + bzero(statistics,sizeof(FrontierStatistics)); + } + +void frontier_statistics_start_debug() + { + if((frontier_log_level==FRONTIER_LOGLEVEL_DEBUG)&&(statistics==NULL)) + { + frontier_statistics_start(); + started_by_debug=1; + } + } + +void frontier_statistics_stop() + { + if(!statistics) + return; + + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__, + "Total queries: %d",statistics->num_queries); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__, + "Total query errors: %d",statistics->num_errors); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__, + "Bytes per query min: %d, avg: %d, max: %d", + statistics->bytes_per_query.min, + statistics->bytes_per_query.avg, + statistics->bytes_per_query.max); + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__, + "Milliseconds per query min: %d, avg: %d, max: %d", + statistics->msecs_per_query.min, + statistics->msecs_per_query.avg, + statistics->msecs_per_query.max); + + frontier_mem_free(statistics); + statistics=NULL; + } + +void frontier_statistics_stop_debug() + { + if(started_by_debug) + { + started_by_debug=0; + frontier_statistics_stop(); + } + } + +int frontier_statistics_get_bytes(FrontierStatistics *stats,int maxbytes) + { + if((stats==NULL)||(maxbytes<=0)||(maxbytes>sizeof(FrontierStatistics))) + return FRONTIER_EIARG; + if(statistics==NULL) + return FRONTIER_ECFG; + + bcopy(statistics,stats,maxbytes); + return FRONTIER_OK; + } + +void frontier_statistics_start_query(fn_query_stat *query_stat) + { + if((statistics==NULL)||(query_stat==NULL)) + return; + if(clock_gettime(CLOCK_REALTIME,&query_stat->starttime)!=0) + query_stat->starttime.tv_sec=0; + } + +static void calcnumstats(FrontierStatisticsNum *num,unsigned int val) + { + double dqueries; + if((num->min==0)||(valmin)) + num->min=val; + if(val>num->max) + num->max=val; + dqueries=statistics->num_queries; + num->avg=round((dqueries*num->avg+val)/(statistics->num_queries+1)); + } + +void frontier_statistics_stop_query(fn_query_stat *query_stat,unsigned int numbytes) + { + unsigned int msec; + struct timespec now; + + if((statistics==NULL)||(query_stat==NULL)||(query_stat->starttime.tv_sec==0)) + return; + if(numbytes==0) + { + statistics->num_errors++; + query_stat->starttime.tv_sec=0; + return; + } + + if(clock_gettime(CLOCK_REALTIME,&now)!=0) + { + query_stat->starttime.tv_sec=0; + return; + } + + msec=round((now.tv_nsec-query_stat->starttime.tv_nsec)/1.0e6); + msec+=(now.tv_sec-query_stat->starttime.tv_sec)*1000; + + frontier_log(FRONTIER_LOGLEVEL_DEBUG,__FILE__,__LINE__, + "query took %u milliseconds and read %u bytes",msec,numbytes); + + calcnumstats(&statistics->msecs_per_query,msec); + calcnumstats(&statistics->bytes_per_query,numbytes); + + statistics->num_queries++; + query_stat->starttime.tv_sec=0; + } diff --git a/test-any.cc b/test-any.cc new file mode 100644 index 0000000..6eb52d3 --- /dev/null +++ b/test-any.cc @@ -0,0 +1,196 @@ +/* + * frontier client test C++ main program that can load any query + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ +#include + +#include +#include +#include +#include + +#ifndef FNTR_USE_EXCEPTIONS +#define CHECK_ERROR() do{if(ds.err_code){std::cout<<"ERROR:"<find_first_of(escape_list,pos); + if(pos>=str->size()) return; + //std::cout<<"pos="<insert(pos,1,'\\'); + pos+=2; + } + } + + +int main(int argc, char **argv) + { + while(1) + { + do_main(argc,argv); + break; + } + } + + +int do_main(int argc, char **argv) + { + //char vc; + int vi; + long long vl; + float vf; + double vd; + std::string *vs; + int arg_name_ind; + +#ifdef FNTR_USE_EXCEPTIONS + try + { +#endif //FNTR_USE_EXCEPTIONS + frontier::init(); + + if(argc<2) + { + std::cout<<"Usage: "< vrq; + vrq.insert(vrq.end(),&metareq); + ds.getData(vrq); + CHECK_ERROR(); + + ds.setCurrentLoad(1); + CHECK_ERROR(); + + int field_num=0; + + std::cout<<"\nObject fields:\n"; + + frontier::AnyData ad; + + // MetaData consists of one record with filed names. + // Let's go over all fields: + while(!ds.isEOR()) + { + std::string *name=ds.getString(); + CHECK_ERROR(); + std::string *type=ds.getString(); + CHECK_ERROR(); + ++field_num; + std::cout< +#include +#include + +int main(int argc, char **argv) + { + //char str[]="select aaa from table"; + //char str[]="select name,version from frontier_descriptors"; + char str[] = "SELECT POS, ID_ID, MITEMS_HCALPEDESTALS_ITEM_MID, MITEMS_HCALPEDESTALS_ITEM_MV_1, MITEMS_HCALPEDESTALS_ITEM_MID, MITEMS_HCALPEDESTALS_ITEM_MV_2, MITEMS_HCALPEDESTALS_ITEM_MID, MITEMS_HCALPEDESTALS_ITEM_MV_3, MITEMS_HCALPEDESTALS_ITEM_MV_4 FROM CMS_VAL_HCAL_POOL_OWNER.HCALPEDESTAL_MITEMS WHERE ID_ID = 1 ORDER BY POS"; + char *buf; + int len; + + frontier_init(malloc,free); + + len=fn_gzip_str2urlenc(str,strlen(str),&buf); + + printf("Len %d res [%s]\n",len,buf); + + return 0; + } diff --git a/test-pescalib.cc b/test-pescalib.cc new file mode 100644 index 0000000..7137306 --- /dev/null +++ b/test-pescalib.cc @@ -0,0 +1,157 @@ +/* + * frontier client pescalib standalone test program + * + * Author: Sergey Kosyakov + * + * $Id$ + * + * Copyright (c) 2009, FERMI NATIONAL ACCELERATOR LABORATORY + * All rights reserved. + * + * For details of the Fermitools (BSD) license see Fermilab-2009.txt or + * http://fermitools.fnal.gov/about/terms.html + * + */ +#include + +#include +#include + +/* +select +address, +gain, +sourcemean, +sourcerms, +sourcenorm, +lasernorm, +linearity1, +linearity2, +linearity3, +attenuation1, +attenuation2, +attenuation3 +from pescalib where cid = ? +*/ + +class PESCalib + { + public: + long long address; + float gain; + float sourcemean; + float sourcerms; + float sourcenorm; + float lasernorm; + float linearity1; + float linearity2; + float linearity3; + float attenuation1; + float attenuation2; + float attenuation3; + + PESCalib(frontier::CDFDataSource& ds) + { + address=ds.getInt(); + gain=ds.getDouble(); + sourcemean=ds.getDouble(); + sourcerms=ds.getDouble(); + sourcenorm=ds.getDouble(); + lasernorm=ds.getDouble(); + linearity1=ds.getDouble(); + linearity2=ds.getDouble(); + linearity3=ds.getDouble(); + attenuation1=ds.getDouble(); + attenuation2=ds.getDouble(); + attenuation3=ds.getDouble(); + } + + void print() + { + std::cout< vrq; + vrq.insert(vrq.end(),&req1); + ds.getData(vrq); + + ds.setCurrentLoad(1); + + int nrec=ds.getRecNum(); + std::cout<<"CID <"< nrec "<< nrec<<'\n'; + + std::vector v_sbp(nrec); + for(int i=0;iprint(); + } + + // Do some usefull things here ... + + // Clean + for(int i=0;i +#include +#include +#include +#include +#include +#include + +int do_main(int argc, char **argv); +static std::string escape_list="\\\'"; +static std::string req_data="frontier_request:1:DEFAULT"; + + +static void str_escape_quota(std::string *str) + { + std::string::size_type pos; + + pos=0; + while(1) + { + pos=str->find_first_of(escape_list,pos); + if(pos==str->npos) return; + //std::cout<<"pos="<insert(pos,1,'\\'); + pos+=2; + } + } + + +int main(int argc, char **argv) + { + while(1) + { + do_main(argc,argv); + break; + } + } + + +static void print_usage(char **argv) + { + std::cout<<"Usage: \n"<(arg_ind+1) && strcmp(argv[arg_ind],"-c")==0) + { + repeat_count=atoi(argv[arg_ind+1]); + arg_ind++; + } + else if(argc>(arg_ind+1) && strcmp(argv[arg_ind],"-F")==0) + { + fork_count=atoi(argv[arg_ind+1]); + arg_ind++; + } + else if(argc>(arg_ind+1) && strcmp(argv[arg_ind],"-f")==0) + file_name=argv[arg_ind+1]; + else if(!file_name && argc>arg_ind) + { + print_usage(argv); + exit(1); + } + arg_ind++; + } + + std::ifstream in_file; + if(file_name) + { + in_file.open(file_name); + if(!in_file.is_open()) + { + std::cout<<"Can not open file "< serverList; + //serverList.push_back("http://lxfs6043.cern.ch:8080/Frontier3D"); + std::list proxyList; + //frontier::DataSource ds(serverList, proxyList); + frontier::Connection con(serverList, proxyList); + + for(idx=0;idx0)&&(idx==fork_count)) + fork(); + + frontier::Session ses(&con); + con.setTimeToLive(ttl); + + frontier::Request req(req_data,frontier::BLOB); + req.addKey("p1",param); + + std::vector vrq; + vrq.push_back(&req); + ses.getData(vrq); + + ses.setCurrentLoad(1); + + int field_num=0; + + std::cout<<"\nObject fields:\n"; + + ses.next(); + // MetaData consists of one record with field names. + // Let's go over all fields: + std::string name,type; + + while(!ses.isEOR()) { + ses.assignString(&name); + ses.assignString(&type); + ++field_num; + std::cout< 1000) + std::cout<<'('< 0) + { + std::cout << std::endl << "Read rate: " << std::endl << std::setprecision(4) + << fstats.bytes_per_query.avg / (float) fstats.msecs_per_query.avg << + " kbytes/sec" << std::endl; + } + frontier_statistics_stop(); + + return 0; + +errexit: + frontier_statistics_stop(); + return 1; + } + +