PVarHandler.cpp

Go to the documentation of this file.

/***************************************************************************
 
  PVarHandler.cpp
 
  Author: Andreas Suter
  e-mail: andreas.suter@psi.ch
 
***************************************************************************/
 
/***************************************************************************
 *   Copyright (C) 2007-2026 by Andreas Suter                              *
 *   andreas.suter@psi.ch                                                  *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program; if not, write to the                         *
 *   Free Software Foundation, Inc.,                                       *
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
 ***************************************************************************/
 
#include <iostream>
#include <fstream>
#include <cstdlib>
#include <string>
 
#include "PVarHandler.h"
 
#include "PSkipper.hpp"
#include "PErrorHandler.hpp"
#include "PStatement.hpp"
#include "PStatementDef.hpp"
#include "PProgram.hpp"
 
#include <boost/spirit/home/x3.hpp>
 
#ifdef MUPP_PYTHON
#include <cmath>
#include <cstring>
#include <cctype>
#include <set>
#include <sstream>
#include <iomanip>
 
#include <TROOT.h>
#include <TSystem.h>
#include <TPython.h>
#endif // MUPP_PYTHON
 
//--------------------------------------------------------------------------
static void mupp_writeErrLog(const std::string &msg)
{
  const char *home = std::getenv("HOME");
  std::string path = std::string(home ? home : ".") + "/.musrfit/mupp/mupp_err.log";
  std::ofstream fout(path.c_str(), std::ios::app);
  if (fout.is_open()) {
    fout << msg << std::endl;
    fout.close();
  }
  std::cerr << msg << std::endl;
}
 
#ifdef MUPP_PYTHON
//--------------------------------------------------------------------------
static std::string mupp_pyNum(double d)
{
  if (std::isnan(d))
    return "float('nan')";
  if (std::isinf(d))
    return (d < 0) ? "float('-inf')" : "float('inf')";
  std::ostringstream os;
  os << std::setprecision(17) << d;
  return os.str();
}
 
//--------------------------------------------------------------------------
static std::string mupp_pyList(const std::vector<double> &v)
{
  std::ostringstream os;
  os << "[";
  for (size_t i=0; i<v.size(); i++) {
    if (i)
      os << ", ";
    os << mupp_pyNum(v[i]);
  }
  os << "]";
  return os.str();
}
 
//--------------------------------------------------------------------------
static std::string mupp_pyStr(const std::string &s)
{
  std::ostringstream os;
  os << "'";
  for (size_t i=0; i<s.size(); i++) {
    if (s[i] == '\\' || s[i] == '\'')
      os << '\\';
    os << s[i];
  }
  os << "'";
  return os.str();
}
 
//--------------------------------------------------------------------------
static std::vector<double> mupp_collValues(PmuppCollection *coll, int pidx)
{
  std::vector<double> v;
  for (int i=0; i<coll->GetNoOfRuns(); i++)
    v.push_back(coll->GetRun(i).GetParam(pidx).GetValue());
  return v;
}
 
//--------------------------------------------------------------------------
static std::vector<double> mupp_collErrors(PmuppCollection *coll, int pidx)
{
  std::vector<double> v;
  for (int i=0; i<coll->GetNoOfRuns(); i++) {
    double p = coll->GetRun(i).GetParam(pidx).GetPosErr();
    double n = coll->GetRun(i).GetParam(pidx).GetNegErr();
    v.push_back(std::sqrt(std::fabs(p*n)));
  }
  return v;
}
 
//--------------------------------------------------------------------------
static bool mupp_isSafePyName(const std::string &s)
{
  if (s.empty())
    return false;
  if (!(std::isalpha((unsigned char)s[0]) || s[0] == '_'))
    return false;
  for (size_t i=0; i<s.size(); i++) {
    if (!(std::isalnum((unsigned char)s[i]) || s[i] == '_'))
      return false;
  }
  static const std::set<std::string> kw = {
    "False","None","True","and","as","assert","async","await","break","class",
    "continue","def","del","elif","else","except","finally","for","from","global",
    "if","import","in","is","lambda","nonlocal","not","or","pass","raise","return",
    "try","while","with","yield"
  };
  return kw.find(s) == kw.end();
}
 
//--------------------------------------------------------------------------
static std::string mupp_indentLines(const std::string &code, const std::string &prefix)
{
  std::ostringstream os;
  std::istringstream is(code);
  std::string line;
  while (std::getline(is, line))
    os << prefix << line << "\n";
  return os.str();
}
 
//--------------------------------------------------------------------------
static bool mupp_getPyString(const std::string &pyExpr, std::string &out)
{
  std::any res;
  std::string cmd = "_anyresult = ROOT.std.make_any['std::string'](" + pyExpr + ")";
  if (!TPython::Exec(cmd.c_str(), &res))
    return false;
  if (!res.has_value())
    return false;
  try {
    out = std::any_cast<std::string>(res);
  } catch (const std::bad_any_cast &) {
    return false;
  }
  return true;
}
 
//--------------------------------------------------------------------------
static std::vector<double> mupp_readPyList(const std::string &name, bool &ok)
{
  ok = false;
  std::vector<double> out;
 
  // serialize on the python side; the try/except guards undefined/non-iterable
  std::string prep =
    "try:\n"
    "    _mupp_s = '1 ' + ' '.join(repr(float(_mupp_x)) for _mupp_x in " + name + ")\n"
    "except Exception:\n"
    "    _mupp_s = '0'\n";
  if (!TPython::Exec(prep.c_str()))
    return out;
 
  std::string s;
  if (!mupp_getPyString("_mupp_s", s))
    return out;
 
  std::istringstream is(s);
  int status = 0;
  if (!(is >> status) || status != 1)
    return out;
 
  double d;
  while (is >> d)
    out.push_back(d);
 
  ok = true;
  return out;
}
#endif // MUPP_PYTHON
 
//--------------------------------------------------------------------------
PVarHandler::PVarHandler() :
  fColl(nullptr), fParseStr(""), fVarName(""), fIsValid(false)
{
  // nothing to be done here
}
 
//--------------------------------------------------------------------------
PVarHandler::PVarHandler(PmuppCollection *coll, std::string parse_str, std::string var_name,
                         const std::vector<PmuppCollection*> &allColl) :
  fColl(coll), fAllColl(allColl), fParseStr(parse_str), fVarName(var_name), fIsValid(false)
{
  // route <python> ... </python> definitions to the embedded Python interpreter
  if (fParseStr.find("<python>") != std::string::npos) {
    evaluatePython();
    return;
  }
 
  injectPredefVariables();
 
  typedef std::string::const_iterator iterator_type;
  iterator_type iter = fParseStr.begin();
  iterator_type end = fParseStr.end();
 
  mupp::PErrorHandler<iterator_type> error_handler(iter, end);   // the error handler
  mupp::prog::PProgram prog(error_handler); // our compiler, and exec
 
  // perform the parsing
  namespace x3 = boost::spirit::x3;
  bool success = x3::phrase_parse(iter, end, mupp::parser::statement_list, mupp::parser::skipper, fAst);
  if (success && iter == end) {
    if (prog(fAst)) { // semantic analysis
      std::vector<double> data, dataErr;
      for (unsigned int i=0; i<fColl->GetRun(0).GetNoOfParam(); i++) {
        data = getData(i);
        dataErr = getDataErr(i);
        prog.add_predef_var_values(getVarName(i), data, dataErr);
      }
      if (!fVarName.empty()) {
        mupp::prog::PProgEval eval(prog.getVars()); // setup evaluation stage
        eval(fAst); // evaluate stuff
 
        // keep data
        bool ok;
        fVar = eval.getVar(fVarName, ok);
        if (!ok) {
          std::cerr << "**ERROR** evalution failed..." << std::endl;
          fIsValid = false;
        }
      }
    }
    fIsValid = true;
  } else {
    std::cerr << "**ERROR** parsing failed..." << std::endl;
  }
}
 
//--------------------------------------------------------------------------
void PVarHandler::evaluatePython()
{
#ifdef MUPP_PYTHON
  // The embedded Python interpreter reads PYTHONPATH at initialization, and
  // TPython's first call imports cppyy/ROOT. Make sure ROOT's library directory
  // (which holds those modules) is on PYTHONPATH before the first TPython call,
  // so the read-back works even when thisroot.sh was not sourced.
  static bool s_pythonPathSet = false;
  if (!s_pythonPathSet) {
    TString pp = TROOT::GetLibDir();
    const char *cur = gSystem->Getenv("PYTHONPATH");
    if (cur != nullptr && cur[0] != '\0') {
      pp += ":";
      pp += cur;
    }
    gSystem->Setenv("PYTHONPATH", pp.Data());
    s_pythonPathSet = true;
  }
 
  // extract the code between the <python> and </python> tags
  const std::string openTag("<python>");
  const std::string closeTag("</python>");
  std::size_t p0 = fParseStr.find(openTag);
  std::size_t p1 = fParseStr.find(closeTag, p0);
  if (p0 == std::string::npos || p1 == std::string::npos) {
    mupp_writeErrLog("**ERROR** python block: missing <python> ... </python> tags.");
    fIsValid = false;
    return;
  }
  std::size_t codeStart = p0 + openTag.size();
  std::string code = fParseStr.substr(codeStart, p1 - codeStart);
 
  // assemble the script: inject all collection parameters, then run the user code
  std::ostringstream py;
  py << "import ROOT\n";   // needed for the make_any based read-back below
  py << "par = {}\n";
  py << "parErr = {}\n";
 
  int nParam = fColl->GetRun(0).GetNoOfParam();
  for (int i=0; i<nParam; i++) {
    std::string name = fColl->GetRun(0).GetParam(i).GetName().toLatin1().data();
    std::vector<double> val = getData(i);
    std::vector<double> err = getDataErr(i);
    py << "par['" << name << "'] = " << mupp_pyList(val) << "\n";
    py << "parErr['" << name << "'] = " << mupp_pyList(err) << "\n";
    if (mupp_isSafePyName(name)) {
      py << name << " = par['" << name << "']\n";
      py << name << "Err = parErr['" << name << "']\n";
    }
  }
 
  // expose every loaded collection explicitly so that a single <python> block
  // can compute (and combine) variables for several collections. coll[<idx>]
  // mirrors the script 'col <idx>' and coll['<name>'] uses the collection name;
  // both keys reference the same per-parameter dict (likewise for collErr).
  py << "coll = {}\n";
  py << "collErr = {}\n";
  for (size_t c=0; c<fAllColl.size(); c++) {
    PmuppCollection *pc = fAllColl[c];
    if (pc == nullptr || pc->GetNoOfRuns() == 0)
      continue;
    py << "_c = {}\n";
    py << "_ce = {}\n";
    int np = pc->GetRun(0).GetNoOfParam();
    for (int j=0; j<np; j++) {
      std::string pname = pc->GetRun(0).GetParam(j).GetName().toLatin1().data();
      py << "_c[" << mupp_pyStr(pname) << "] = " << mupp_pyList(mupp_collValues(pc, j)) << "\n";
      py << "_ce[" << mupp_pyStr(pname) << "] = " << mupp_pyList(mupp_collErrors(pc, j)) << "\n";
    }
    std::string cname = pc->GetName().toLatin1().data();
    py << "coll[" << c << "] = _c\n";
    py << "collErr[" << c << "] = _ce\n";
    py << "coll[" << mupp_pyStr(cname) << "] = _c\n";
    py << "collErr[" << mupp_pyStr(cname) << "] = _ce\n";
  }
 
  // run the user code inside a try/except so we can capture the traceback
  py << "import traceback as _mupp_tb\n";
  py << "_mupp_err = ''\n";
  py << "try:\n";
  py << mupp_indentLines(code, "    ");
  py << "    pass\n";
  py << "except Exception:\n";
  py << "    _mupp_err = _mupp_tb.format_exc()\n";
 
  if (!TPython::Exec(py.str().c_str())) {
    mupp_writeErrLog("**ERROR** python block failed to execute "
                     "(syntax error or interpreter problem). See stderr for details.");
    fIsValid = false;
    return;
  }
 
  // did the user code raise at run time?
  std::string errStr;
  if (mupp_getPyString("_mupp_err", errStr) && !errStr.empty()) {
    mupp_writeErrLog(std::string("**ERROR** python evaluation failed:\n") + errStr);
    fIsValid = false;
    return;
  }
 
  // check-only mode (no variable requested): success if the code ran
  if (fVarName.empty()) {
    fIsValid = true;
    return;
  }
 
  // read back value and (user-supplied) error
  bool okVal = false, okErr = false;
  std::vector<double> val = mupp_readPyList(fVarName, okVal);
  std::vector<double> err = mupp_readPyList(fVarName + "Err", okErr);
 
  if (!okVal) {
    mupp_writeErrLog(std::string("**ERROR** python block did not define an iterable variable '") + fVarName + "'.");
    fIsValid = false;
    return;
  }
 
  int nRuns = fColl->GetNoOfRuns();
  if ((int)val.size() != nRuns) {
    std::ostringstream msg;
    msg << "**ERROR** python variable '" << fVarName << "' has " << val.size()
        << " values, but the collection has " << nRuns << " runs.";
    mupp_writeErrLog(msg.str());
    fIsValid = false;
    return;
  }
 
  // the error is user supplied; default to zeros if not provided / size mismatch
  if (!okErr || err.size() != val.size())
    err.assign(val.size(), 0.0);
 
  fVar.SetValue(val);
  fVar.SetError(err);
  fIsValid = true;
#else
  mupp_writeErrLog("**ERROR** this mupp build has no Python support; "
                   "rebuild ROOT with -Dtpython=ON to use <python> variable blocks.");
  fIsValid = false;
#endif // MUPP_PYTHON
}
 
//--------------------------------------------------------------------------
std::vector<double> PVarHandler::getValues()
{
  std::vector<double> data;
  if (fIsValid)
    data = fVar.GetValue();
 
  return data;
}
 
//--------------------------------------------------------------------------
std::vector<double> PVarHandler::getErrors()
{
  std::vector<double> data;
  if (fIsValid)
    data = fVar.GetError();
 
  return data;
}
 
//--------------------------------------------------------------------------
void PVarHandler::injectPredefVariables()
{
  mupp::ast::statement var_stat;
  mupp::ast::variable_declaration var;
 
  std::string varName, errVarName;
  PmuppRun run = fColl->GetRun(0);
  for (int i=0; i<run.GetNoOfParam(); i++) {
    varName = run.GetParam(i).GetName().toLatin1().data();
    errVarName = varName + "Err";
    // inject err_name
    var.lhs.name = errVarName;
    var_stat = var;
    fAst.push_front(var_stat);
    // inject var_name
    var.lhs.name = varName;
    var_stat = var;
    fAst.push_front(var_stat);
  }
}
 
//--------------------------------------------------------------------------
std::string PVarHandler::getVarName(int idx)
{
  std::string name("??");
 
  // make sure idx is in range
  if (idx >= fColl->GetRun(0).GetNoOfParam())
    return name;
 
  return fColl->GetRun(0).GetParam(idx).GetName().toLatin1().data();
}
 
//--------------------------------------------------------------------------
std::vector<double> PVarHandler::getData(int idx)
{
  std::vector<double> data;
 
  // make sure idx is in range
  if (idx >= fColl->GetRun(0).GetNoOfParam())
    return data;
 
  double dval;
  for (int i=0; i<fColl->GetNoOfRuns(); i++) {
    dval = fColl->GetRun(i).GetParam(idx).GetValue();
    data.push_back(dval);
  }
 
  return data;
}
 
//--------------------------------------------------------------------------
std::vector<double> PVarHandler::getDataErr(int idx)
{
  std::vector<double> err;
 
  // make sure idx is in range
  if (idx >= fColl->GetRun(0).GetNoOfParam())
    return err;
 
  double dvalPos, dvalNeg;
  for (int i=0; i<fColl->GetNoOfRuns(); i++) {
    dvalPos = fColl->GetRun(i).GetParam(idx).GetPosErr();
    dvalNeg = fColl->GetRun(i).GetParam(idx).GetNegErr();
    dvalPos = sqrt(fabs(dvalPos*dvalNeg)); // geometric mean of pos/neg error
    err.push_back(dvalPos);
  }
 
  return err;
}