Merge branch 'master' into 'master'

v1.8.12.0.1 update

Closes QPR-12520 and QPR-12398

See merge request qs/ore-github!57

Docs/UserGuide/tradecomponents/scheduledata.tex

@@ -38,6 +38,7 @@ \subsubsection{Schedule Data (Rules, Dates and Derived)}\label{ss:schedule_data}
                   <Convention>Following</Convention>
                   <Tenor>3M</Tenor>
                   <EndOfMonth>false</EndOfMonth>
+                  <EndOfMOnthConvention>Following</EndOfMOnthConvention>
                   <Dates>
                     <Date>2012-01-06</Date>
                     <Date>2012-04-10</Date>
@@ -129,6 +130,14 @@ \subsubsection{Schedule Data (Rules, Dates and Derived)}\label{ss:schedule_data}
 
 Allowable values: Boolean node, allowing \emph{Y, N, 1, 0, true, false} etc. The full set of allowable values is given in Table \ref{tab:boolean_allowable}. Defaults to \emph{false} if left blank or omitted. Must be set to \emph{false} or omitted if the date generation Rule is set to \emph{CDS} or \emph{CDS2015}.
 
+\item \lstinline!EndOfMonthConvention! [Optional]: Determines the adjustment of the end-of-month schedule dates with regards to the selected calendar.
+This field is only used when \lstinline!EndOfMonth! is true. If left blank or omitted, then the default \lstinline!Preceding! convention is applied
+(i.e.\ end-of-month dates will never be adjusted over to the beginning of the next month)
+
+Allowable values: See Table \ref{tab:convention} Roll Convention.
+
+Allowable values: Boolean node, allowing \emph{Y, N, 1, 0, true, false} etc. The full set of allowable values is given in Table \ref{tab:boolean_allowable}. Defaults to \emph{false} if left blank or omitted. Must be set to \emph{false} or omitted if the date generation Rule is set to \emph{CDS} or \emph{CDS2015}.
+
 \item \lstinline!FirstDate! [Optional]: Date for initial stub period. For date generation rules \emph{CDS} and \emph{CDS2015}, if given, this
   overwrites the first date of the schedule that is otherwise built from IMM dates.
 
@@ -178,6 +187,10 @@ \subsubsection{Schedule Data (Rules, Dates and Derived)}\label{ss:schedule_data}
 
 Allowable values: Boolean node, allowing \emph{Y, N, 1, 0, true, false} etc. The full set of allowable values is given in Table \ref{tab:boolean_allowable}. Defaults to \emph{false} if left blank or omitted.
 
+\item \lstinline!EndOfMonthConvention! [Optional]: Whenever the \lstinline!EndOfMonth! logic is applied, this is used as the roll convention along with the \lstinline!Calendar!for any date adjustments.
+
+Allowable values: See Table \ref{tab:convention} Roll Convention. Defaults to \emph{Preceding} if omitted.
+
 \item \lstinline!Dates!: This is a sub-sub-node and contains child elements of type
   \lstinline!Date!. In this case the schedule dates are determined
   directly by the \lstinline!Date! child elements.  At least two

Examples/Example_24/ExpectedOutput/flows.csv

@@ -1 +1,9 @@
-#TradeId,Type,CashflowNo,LegNo,PayDate,FlowType,Amount,Currency,Coupon,Accrual,AccrualStartDate,AccrualEndDate,AccruedAmount,fixingDate,fixingValue,Notional,DiscountFactor,PresentValue,FXRate(Local-Base),PresentValue(Base),BaseCurrency,FloorStrike,CapStrike,FloorVolatility,CapVolatility
+#TradeId,Type,CashflowNo,LegNo,PayDate,FlowType,Amount,Currency,Coupon,Accrual,AccrualStartDate,AccrualEndDate,AccruedAmount,fixingDate,fixingValue,Notional,DiscountFactor,PresentValue,FXRate(Local-Base),PresentValue(Base),BaseCurrency,FloorStrike,CapStrike,FloorVolatility,CapVolatility,EffectiveFloorVolatility,EffectiveCapVolatility
+CommodityForward_Gold_1Y,CommodityForward,1,0,#N/A,Notional,-1160593.3333,USD,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,0.9943418517,-1154026.5240929599,1.0000000000,-1154026.5240929599,USD,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A
+CommodityForward_Gold_1Y,CommodityForward,1,1,#N/A,Notional,1162000.0000,USD,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,0.9943418517,1155425.2316309642,1.0000000000,1155425.2316309642,USD,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A
+CommodityForward_Gold_Apr_17,CommodityForward,1,0,#N/A,Notional,580900.0000,USD,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,0.9928024234,576718.9277435583,1.0000000000,576718.9277435583,USD,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A
+CommodityForward_Gold_Apr_17,CommodityForward,1,1,#N/A,Notional,-580900.0000,USD,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,0.9928024234,-576718.9277435583,1.0000000000,-576718.9277435583,USD,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A
+CommodityForward_WTI_2Y,CommodityForward,1,0,#N/A,Notional,44308571.4286,USD,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,0.9871068084,43737292.5278331637,1.0000000000,43737292.5278331637,USD,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A
+CommodityForward_WTI_2Y,CommodityForward,1,1,#N/A,Notional,-46000000.0000,USD,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,0.9871068084,-45406913.1866207123,1.0000000000,-45406913.1866207123,USD,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A
+CommodityForward_WTI_Oct_21,CommodityForward,1,0,#N/A,Notional,-24875000.0000,USD,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,0.9432249101,-23462719.6376393437,1.0000000000,-23462719.6376393437,USD,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A
+CommodityForward_WTI_Oct_21,CommodityForward,1,1,#N/A,Notional,24875000.0000,USD,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A,0.9432249101,23462719.6376393437,1.0000000000,23462719.6376393437,USD,#N/A,#N/A,#N/A,#N/A,#N/A,#N/A

OREAnalytics/orea/app/inputparameters.cpp

@@ -192,6 +192,11 @@ void InputParameters::setScenarioSimMarketParamsFromFile(const std::string& file
     scenarioSimMarketParams_->fromFile(fileName);
 }
 
+void InputParameters::setHistVarSimMarketParams(const std::string& xml) {
+    histVarSimMarketParams_ = QuantLib::ext::make_shared<ScenarioSimMarketParameters>();
+    histVarSimMarketParams_->fromXMLString(xml);
+}
+
 void InputParameters::setHistVarSimMarketParamsFromFile(const std::string& fileName) {
     histVarSimMarketParams_ = QuantLib::ext::make_shared<ScenarioSimMarketParameters>();
     histVarSimMarketParams_->fromFile(fileName);

OREAnalytics/orea/app/inputparameters.hpp

@@ -185,6 +185,7 @@ class InputParameters {
     void setBenchmarkVarPeriod(const std::string& period);
     void setHistoricalScenarioReader(const std::string& fileName);
     void setSensitivityStreamFromBuffer(const std::string& buffer);
+    void setHistVarSimMarketParams(const std::string& xml);
     void setHistVarSimMarketParamsFromFile(const std::string& fileName);
     void setOutputHistoricalScenarios(const bool b) { outputHistoricalScenarios_ = b; }
 

OREAnalytics/orea/cube/sensitivitycube.cpp

@@ -156,6 +156,16 @@ RiskFactorKey SensitivityCube::upDownFactor(const Size index) const {
     }
 }
 
+SensitivityCube::FactorData SensitivityCube::upThenDownFactorData(const RiskFactorKey& rfkey) {
+    if (auto f = upFactors_.find(rfkey); f != upFactors_.end())
+        return f->second;
+    else if (auto f = downFactors_.find(rfkey); f != downFactors_.end())
+        return f->second;
+    else {
+        QL_FAIL("SensitivityCube::upThenDownFactorData(): no up or down factor data found for " << rfkey);
+    }
+}
+
 SensitivityCube::crossPair SensitivityCube::crossFactor(const Size crossIndex) const {
     if (auto k = crossIndexToKey_.find(crossIndex); k != crossIndexToKey_.end()) {
         return k->second;

OREAnalytics/orea/cube/sensitivitycube.hpp

@@ -101,6 +101,9 @@ class SensitivityCube {
     //! Return the map of down risk factors to its factor data
     const std::map<RiskFactorKey, SensitivityCube::FactorData>& downFactors() const { return downFactors_; };
 
+    //! Return the factor data for an up shift of a rf key, if that does not exist for a down shift of the same rf key
+    SensitivityCube::FactorData upThenDownFactorData(const RiskFactorKey& rfkey);
+
     //! Returns the set of pairs of risk factor keys for which a cross gamma is available
     const std::map<crossPair, std::tuple<SensitivityCube::FactorData, SensitivityCube::FactorData, QuantLib::Size>>&
     crossFactors() const;

OREAnalytics/orea/engine/sensitivitycubestream.cpp

@@ -84,7 +84,7 @@ SensitivityRecord SensitivityCubeStream::next() {
     sr.baseNpv = cubes_[currentCubeIdx_]->npv(tradeIdx);
 
     if (currentDeltaKey_ != currentDeltaKeys_.end()) {
-        auto fd = cubes_[currentCubeIdx_]->upFactors().at(*currentDeltaKey_);
+        auto const& fd = cubes_[currentCubeIdx_]->upThenDownFactorData(*currentDeltaKey_);
         sr.key_1 = *currentDeltaKey_;
         sr.desc_1 = fd.factorDesc;
         sr.shift_1 = fd.targetShiftSize;
@@ -95,7 +95,7 @@ SensitivityRecord SensitivityCubeStream::next() {
             sr.gamma = Null<Real>();
         ++currentDeltaKey_;
     } else if (currentCrossGammaKey_ != currentCrossGammaKeys_.end()) {
-        auto fd = cubes_[currentCubeIdx_]->crossFactors().at(*currentCrossGammaKey_);
+        auto const& fd = cubes_[currentCubeIdx_]->crossFactors().at(*currentCrossGammaKey_);
         sr.key_1 = currentCrossGammaKey_->first;
         sr.desc_1 = std::get<0>(fd).factorDesc;
         sr.shift_1 = std::get<0>(fd).targetShiftSize;

OREAnalytics/test/cube.cpp

@@ -53,6 +53,8 @@
 #include <ql/time/daycounters/actualactual.hpp>
 #include <oret/toplevelfixture.hpp>
 #include <test/oreatoplevelfixture.hpp>
+#include <ored/report/inmemoryreport.hpp>
+#include <orea/app/reportwriter.hpp>
 
 #include "testmarket.hpp"
 
@@ -537,6 +539,49 @@ BOOST_AUTO_TEST_CASE(testDoublePrecisionJaggedCube) {
     IndexManager::instance().clearHistories();
 }
 
+string writeCube(const QuantLib::ext::shared_ptr<NPVCube>& cube, Size bufferSize) {
+    auto report = QuantLib::ext::make_shared<InMemoryReport>(bufferSize);
+    ReportWriter().writeCube(*report, cube);
+    string fileName = boost::filesystem::unique_path().string();
+    report->toFile(fileName);
+    return fileName;
+}
+
+void diffFiles(string filename1, string filename2) {
+    std::ifstream ifs1(filename1);
+    std::ifstream ifs2(filename2);
+
+    std::istream_iterator<char> b1(ifs1), e1;
+    std::istream_iterator<char> b2(ifs2), e2;
+
+    BOOST_CHECK_EQUAL_COLLECTIONS(b1, e1, b2, e2);
+}
+
+// Test the functionality of class InMemoryReport to cache data on disk
+BOOST_AUTO_TEST_CASE(testInMemoryReportBuffer) {
+
+    // Generate a cube
+    std::set<string> ids{string("id")}; // the overlap doesn't matter
+    vector<Date> dates(50, Date());
+    Size samples = 200;
+    Size depth = 6;
+    auto c = QuantLib::ext::make_shared<SinglePrecisionInMemoryCubeN>(Date(), ids, dates, samples, depth);
+
+    // From the cube, generate multiple copies of the report, each of which which will have ~60K rows.
+    // Specify different values for the buffer size in InMemoryReport:
+    string filename_0 = writeCube(c, 0);            // no buffering
+    string filename_100 = writeCube(c, 100);
+    string filename_1000 = writeCube(c, 1000);
+    string filename_10000 = writeCube(c, 10000);
+    string filename_100000 = writeCube(c, 100000);  // buffer size > report size, resulting in no buffering
+
+    // Verify that buffering generates the same output as no buffering
+    diffFiles(filename_0, filename_100);
+    diffFiles(filename_0, filename_1000);
+    diffFiles(filename_0, filename_10000);
+    diffFiles(filename_0, filename_100000);
+}
+
 BOOST_AUTO_TEST_SUITE_END()
 
 BOOST_AUTO_TEST_SUITE_END()

OREData/ored/configuration/curveconfigurations.cpp

@@ -235,8 +235,8 @@ std::set<string> CurveConfigurations::quotes(const QuantLib::ext::shared_ptr<Tod
             if (spec->baseType() == CurveSpec::CurveType::FX) {
                 QuantLib::ext::shared_ptr<FXSpotSpec> fxss = QuantLib::ext::dynamic_pointer_cast<FXSpotSpec>(spec);
                 QL_REQUIRE(fxss, "Expected an FXSpotSpec but did not get one");
-                string strQuote = "FX/RATE/" + fxss->unitCcy() + "/" + fxss->ccy();
-                quotes.insert(strQuote);
+                quotes.insert("FX/RATE/" + fxss->unitCcy() + "/" + fxss->ccy());
+                quotes.insert("FX/RATE/" + fxss->ccy() + "/" + fxss->unitCcy());
             }
         }
     }

OREData/ored/configuration/yieldcurveconfig.cpp

@@ -235,8 +235,15 @@ const vector<string>& YieldCurveConfig::quotes() {
             // Check if the segment is a CrossCcyYieldCurveSegment and add the FX spot rate to the
             // set of quotes needed for the YieldCurveConfig if it has not already been added.
             if (auto xccySegment = QuantLib::ext::dynamic_pointer_cast<CrossCcyYieldCurveSegment>(c)) {
-                if (!addedFxSpot)
+                if (!addedFxSpot) {
                     quotes_.push_back(xccySegment->spotRateID());
+                    // we add the inverted pair as well, because the original pair might get removed from the market
+                    // data loader if both are present in the input market data
+                    if (auto md = boost::dynamic_pointer_cast<FXSpotQuote>(
+                            parseMarketDatum(Date(), xccySegment->spotRateID(), 1.0))) {
+                        quotes_.push_back("FX/RATE/" + md->ccy() + "/" + md->unitCcy());
+                    }
+                }
             }
         }
     }

OREData/ored/portfolio/collateralbalance.cpp

@@ -30,10 +30,11 @@ using ore::data::parseReal;
 namespace ore {
 namespace data {
 
-void CollateralBalances::add(const QuantLib::ext::shared_ptr<CollateralBalance>& cb) {
-    std::pair<NettingSetDetails, QuantLib::ext::shared_ptr<CollateralBalance>> newCollateralBalance(cb->nettingSetDetails(),
-                                                                                            cb);
-    collateralBalances_.insert(newCollateralBalance);
+void CollateralBalances::add(const QuantLib::ext::shared_ptr<CollateralBalance>& cb, const bool overwrite) {
+    if (collateralBalances_.find(cb->nettingSetDetails()) != collateralBalances_.end() && !overwrite)
+        QL_FAIL("Cannot add collateral balances since it already exists and overwrite=false: " << cb->nettingSetDetails());
+
+    collateralBalances_[cb->nettingSetDetails()] = cb;
 }
 
 bool CollateralBalances::has(const NettingSetDetails& nettingSetDetails) const {
@@ -84,8 +85,8 @@ void CollateralBalance::fromXML(XMLNode* node) {
     if (nettingSetDetailsNode) {
         nettingSetDetails_.fromXML(nettingSetDetailsNode);
     } else {
-        nettingSetId_ = XMLUtils::getChildValue(node, "NettingSetId", false);
-        nettingSetDetails_ = NettingSetDetails(nettingSetId_);
+        const string nettingSetId = XMLUtils::getChildValue(node, "NettingSetId", false);
+        nettingSetDetails_ = NettingSetDetails(nettingSetId);
     }   
 
     currency_ = XMLUtils::getChildValue(node, "Currency", true);
@@ -111,8 +112,8 @@ void CollateralBalance::fromXML(XMLNode* node) {
 XMLNode* CollateralBalance::toXML(XMLDocument& doc) const {
     XMLNode* node = doc.allocNode("CollateralBalance");
     XMLUtils::addChild(doc, node, "Currency", currency_);
-    if (nettingSetDetails_.empty()) {
-        XMLUtils::addChild(doc, node, "NettingSetId", nettingSetId_);
+    if (nettingSetDetails_.emptyOptionalFields()) {
+        XMLUtils::addChild(doc, node, "NettingSetId", nettingSetDetails_.nettingSetId());
     } else {
         XMLUtils::appendNode(node, nettingSetDetails_.toXML(doc));
     }

OREData/ored/portfolio/collateralbalance.hpp

@@ -39,26 +39,25 @@ class CollateralBalance : public ore::data::XMLSerializable {
       default constructor
     */
     CollateralBalance()
-        : nettingSetId_(""), nettingSetDetails_(NettingSetDetails()), currency_(""), 
+        : nettingSetDetails_(NettingSetDetails()), currency_(""), 
           im_(QuantLib::Null<QuantLib::Real>()), vm_(QuantLib::Null<QuantLib::Real>()) {}
 
     CollateralBalance(ore::data::XMLNode* node);
 
-    CollateralBalance(const std::string& nettingSetId, const std::string& currency,
+    CollateralBalance(const NettingSetDetails& nettingSetDetails, const std::string& currency,
                       const QuantLib::Real& im, const QuantLib::Real& vm = QuantLib::Null<QuantLib::Real>())
-        : nettingSetId_(nettingSetId), nettingSetDetails_(NettingSetDetails()), currency_(currency),
-          im_(im), vm_(vm) {}
+         : nettingSetDetails_(nettingSetDetails), currency_(currency), im_(im), vm_(vm) {}
 
-    CollateralBalance(const NettingSetDetails& nettingSetDetails, const std::string& currency,
+    CollateralBalance(const std::string& nettingSetId, const std::string& currency,
                       const QuantLib::Real& im, const QuantLib::Real& vm = QuantLib::Null<QuantLib::Real>())
-         : nettingSetId_(""), nettingSetDetails_(nettingSetDetails), currency_(currency), im_(im), vm_(vm) {}
+        : CollateralBalance(NettingSetDetails(nettingSetId), currency, im, vm) {}
 
     void fromXML(ore::data::XMLNode* node) override;
     ore::data::XMLNode* toXML(ore::data::XMLDocument& doc) const override;
 
     // Getters
     const std::string& nettingSetId() const {
-        return (nettingSetDetails_.empty() ? nettingSetId_ : nettingSetDetails_.nettingSetId());
+        return nettingSetDetails_.nettingSetId();
     }
     const NettingSetDetails nettingSetDetails() const { return nettingSetDetails_; }
     const std::string& currency() const { return currency_; }
@@ -70,7 +69,6 @@ class CollateralBalance : public ore::data::XMLSerializable {
     QuantLib::Real& variationMargin() { return vm_; }
 
 private:
-    std::string nettingSetId_;
     NettingSetDetails nettingSetDetails_;
     std::string currency_;
     QuantLib::Real im_, vm_;
@@ -112,7 +110,7 @@ class CollateralBalances : public ore::data::XMLSerializable {
     /*!
       adds a new collateral balance to manager
     */
-    void add(const QuantLib::ext::shared_ptr<CollateralBalance>& cb);
+    void add(const QuantLib::ext::shared_ptr<CollateralBalance>& cb, const bool overwrite = false);
 
     /*!
       extracts a collateral balance from manager

OREData/ored/portfolio/commodityswap.cpp

@@ -197,6 +197,7 @@ const std::map<std::string,boost::any>& CommoditySwap::additionalData() const {
                     additionalData_["quantity[" + label + "]"] = indexedFlow->quantity();
                     additionalData_["periodQuantity[" + label + "]"] = indexedFlow->periodQuantity();
                     additionalData_["gearing[" + label + "]"] = indexedFlow->gearing();
+                    additionalData_["spread[" + label + "]"] = indexedFlow->spread();
                     if (indexedFlow->isAveragingFrontMonthCashflow(asof)) {
                         std::vector<Real> priceVec;
                         std::vector<std::string> indexVec;
@@ -242,6 +243,7 @@ const std::map<std::string,boost::any>& CommoditySwap::additionalData() const {
                     additionalData_["quantity[" + label + "]"] = indexedAvgFlow->quantity();
                     additionalData_["periodQuantity[" + label + "]"] = indexedAvgFlow->periodQuantity();
                     additionalData_["gearing[" + label + "]"] = indexedAvgFlow->gearing();
+                    additionalData_["spread[" + label + "]"] = indexedAvgFlow->spread();
                     std::vector<Real> priceVec;
                     std::vector<std::string> indexVec;
                     std::vector<Date> indexExpiryVec, pricingDateVec;

OREData/ored/portfolio/legdata.cpp

@@ -942,7 +942,9 @@ Leg makeSimpleLeg(const LegData& data) {
     Leg leg;
     for (Size i = 0; i < dates.size(); i++) {
         Date d = parseDate(dates[i]);
-        leg.push_back(QuantLib::ext::shared_ptr<CashFlow>(new SimpleCashFlow(amounts[i], d)));
+        if (!data.paymentCalendar().empty() && !data.paymentConvention().empty())
+            d = parseCalendar(data.paymentCalendar()).adjust(d, parseBusinessDayConvention(data.paymentConvention()));
+        leg.push_back(QuantLib::ext::make_shared<SimpleCashFlow>(amounts[i], d));
     }
     return leg;
 }

OREData/ored/portfolio/schedule.cpp

@@ -128,6 +128,7 @@ void ScheduleDates::fromXML(XMLNode* node) {
     tenor_ = XMLUtils::getChildValue(node, "Tenor") == "1T" ? "0D" : XMLUtils::getChildValue(node, "Tenor");
     was1T_ = XMLUtils::getChildValue(node, "Tenor") == "1T" ? true : false;
     endOfMonth_ = XMLUtils::getChildValue(node, "EndOfMonth");
+    endOfMonthConvention_ = XMLUtils::getChildValue(node, "EndOfMonthConvention");
     dates_ = XMLUtils::getChildrenValues(node, "Dates", "Date");
 }
 
@@ -139,6 +140,8 @@ XMLNode* ScheduleDates::toXML(XMLDocument& doc) const {
     XMLUtils::addChild(doc, node, "Tenor", was1T_ ? "1T" : tenor_);
     if (!endOfMonth_.empty())
         XMLUtils::addChild(doc, node, "EndOfMonth", endOfMonth_);
+    if (!endOfMonthConvention_.empty())
+        XMLUtils::addChild(doc, node, "EndOfMonthConvention", endOfMonthConvention_);
     XMLUtils::addChildren(doc, node, "Dates", "Date", dates_);
     return node;
 }