{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests as req\n",
    "import numpy\n",
    "import pandas as pd\n",
    "import json\n",
    "import quantipy as qp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Configuration\n",
    "idp_url = \"https://idp.euro.confirmit.com/\"\n",
    "site_url = \"https://ws.euro.confirmit.com/\"\n",
    "client_id =  \"aa55d014-829c-454b-aaf9-dc4549d9fa80\"\n",
    "client_secret = \"788e2edd-6084-4094-9b9d-1e0dc1979129\"\n",
    "\n",
    "# Source configuration\n",
    "surveyid = \"p12345\" #Webinar Demo: Relationship survey\n",
    "\n",
    "# Target configuration\n",
    "hubid = \"1234\" #https://author.euro.confirmit.com/hub/269319/overview\n",
    "hub_tableid_basic = \"2\" #weights_p12345_w0_responseid\n",
    "hub_tableid_advanced = \"3\" #weights_p12345_w1_responseid\n",
    "hub_tableid_cellweights = \"4\" #weights_p12345_w1_cell\n",
    "hub_tableid_interlocked = \"5\" #weights_p12345_w0_age_gender_interlocked"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Alternative 1: Retrieve Data using the API directly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get access token\n",
    "response = req.post(idp_url + 'identity/connect/token', \n",
    "                    data = \"grant_type=api-user&scope=pub.surveys pub.hubs\", \n",
    "                    auth=(client_id, client_secret), \n",
    "                    headers = {'Content-Type': 'application/x-www-form-urlencoded'}) \n",
    "token = response.json()['access_token']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get source data records\n",
    "headers = {'Authorization': 'Bearer ' + source_token, \"Accept\": \"application/x-ndjson\", \"Content-Type\": \"application/json\"} \n",
    "url = site_url + 'v1/surveys/' + surveyid + '/responses/data'\n",
    "response = req.get(url, data = None, headers = headers, stream=False) \n",
    "response.raise_for_status()\n",
    "\n",
    "# Decode json response\n",
    "res = response.content.decode(\"utf-8\")\n",
    "json_lines = res.splitlines()\n",
    "json_data = []\n",
    "for line in json_lines:\n",
    "    json_data.append(json.loads(line))\n",
    "\n",
    "# Normalize the data in a pandas dataframe\n",
    "df = pd.io.json.json_normalize(json_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read data from pandas dataframe into quantipy dataset\n",
    "dataset = qp.DataSet(name='test', dimensions_comp=True)\n",
    "\n",
    "# Column 'name' is not allowed by quantipy\n",
    "dataset.from_components(df.drop(columns=['name']))\n",
    "\n",
    "# All columns are of type string. convert appropriate columns to singles to allow weighting\n",
    "dataset.convert('fiscal_year', 'single')\n",
    "dataset.convert('gender', 'single')\n",
    "dataset.convert('age_group', 'single')\n",
    "dataset.convert('region', 'single')\n",
    "dataset.convert('continent', 'single')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Alternative 2: Retrieve Data using Quantipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = qp.DataSet(\"test2\")\n",
    "dataset.read_forsta_api(projectid, site_url, idp_url, client_id, client_secret)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define and generate weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weighting basic rim weights\n",
    "basicScheme = qp.Rim('test_basic')\n",
    "\n",
    "# Targets\n",
    "gender_targets = {}\n",
    "gender_targets['gender'] = {1: 30, 2: 70}\n",
    "age_group_targets = {}\n",
    "age_group_targets['age_group'] = {1: 10, 2: 15, 3:10, 4: 35, 5: 30}\n",
    "region_targets = {}\n",
    "region_targets = {'region': {1: 10, 2: 15, 3:15, 4: 25, 5: 15, 6: 10,7: 10}}\n",
    "all_targets = [gender_targets, age_group_targets, region_targets]\n",
    "basicScheme.set_targets(targets=all_targets, group_name='basic weights')\n",
    "basicScheme.groups['basic weights']['targets']\n",
    "\n",
    "# Calculate weights\n",
    "responseweights_basic = dataset.weight(basicScheme, weight_name='weight', unique_key='responseid', inplace=False)\n",
    "responseweights_basic = responseweights_basic.reindex(['responseid', 'weight'],axis='columns')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Weighting with different weights within groups, and weighting on groups themselves\n",
    "advancedScheme = qp.Rim('test_advanced')\n",
    "\n",
    "# Targets within groups\n",
    "gender_target_fy1 = {'gender': {1:40, 2:60}}\n",
    "gender_target_fy2and3 = {'gender': {1:55, 2:45}}\n",
    "AgeGroup_target_fy1 = {'age_group': {1: 10, 2: 15, 3:10, 4: 35, 5: 30}}\n",
    "AgeGroup_target_fy2and3 = {'age_group': {1: 20, 2: 10, 3:15, 4: 20, 5: 35}}\n",
    "region_target_fy1 = {'region': {1: 5, 2: 10, 3:5, 4: 25, 5: 30, 6: 10, 7: 15}}\n",
    "region_target_fy2and3 = {'region': {1: 10, 2: 15, 3:15, 4: 25, 5: 15, 6: 10,7: 10}}\n",
    "all_targets_fy1 = [gender_target_fy1, AgeGroup_target_fy1, region_target_fy1]\n",
    "all_targets_fy2and3 = [gender_target_fy2and3, AgeGroup_target_fy2and3, region_target_fy2and3]\n",
    "\n",
    "# Add fiscal year groups\n",
    "filter_fy1 = 'fiscal_year == 1'\n",
    "filter_fy2and3 = 'fiscal_year == 2 or fiscal_year == 3'\n",
    "advancedScheme.add_group(name='fiscal year 1', filter_def=filter_fy1, targets=all_targets_fy1)\n",
    "advancedScheme.add_group(name='fiscal years 2 and 3', filter_def=filter_fy2and3, targets=all_targets_fy2and3)\n",
    "\n",
    "# Add targets for the fiscal year groups\n",
    "fy_targets = {'fiscal year 1': 40, 'fiscal years 2 and 3': 60}\n",
    "advancedScheme.group_targets(fy_targets)\n",
    "\n",
    "# Calculate weights\n",
    "responseweights_adv = dataset.weight(advancedScheme, weight_name='weight', unique_key='responseid',inplace=False)\n",
    "responseweights_adv = responseweights_adv.reindex(['responseid', 'weight'],axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create set with only cellweights for advanced scheme (fiscal year groups)\n",
    "wdf = dataset.weight(advancedScheme, weight_name='weight', unique_key='responseid',inplace=False)\n",
    "wdf = wdf.drop(columns=['responseid'])\n",
    "\n",
    "wdfdistinct = wdf.drop_duplicates(subset=['fiscal_year', 'gender', 'age_group', 'region', 'weight'], inplace=False).reindex(['fiscal_year', 'gender', 'age_group', 'region', 'weight'],axis='columns').sort_values(by=['fiscal_year', 'gender', 'age_group', 'region'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create interlocked variable in dataset ('age_group' interlocked with 'gender')\n",
    "interlocked_dataset = dataset.clone()\n",
    "interlocked_dataset.interlock(\"age_gender\", \"age and gender\", ['age_group','gender'], '/')\n",
    "interlocked_dataset.convert('age_gender', 'single')\n",
    "\n",
    "# set interlocked cell targets (cells correspond to [age_group 1, gender 1], [age_group 1, gender 2], [age_group 2, gender 1], ...)\n",
    "interlockedScheme = qp.Rim('test_interlocked')\n",
    "age_gender_target = {'age_gender': {1: 15, 2: 15, 3:10, 4: 5, 5: 10, 6: 15, 7: 10, 8: 5, 9: 10, 10: 5 }}\n",
    "interlockedScheme.set_targets(targets=age_gender_target, group_name='interlocked weights')\n",
    "\n",
    "# Calculate weights\n",
    "interlocked_dataset.weight(interlockedScheme, weight_name='weight', unique_key='responseid',inplace=True)\n",
    "interlockedweights = interlocked_dataset.subset(['age_group', 'gender', 'weight'], inplace=False).data().drop_duplicates(subset=['age_group', 'gender'], inplace=False).reindex(['age_group', 'gender', 'weight'],axis='columns').sort_values(by=['age_group'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Insert weights in custom tables in hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert to json\n",
    "class Obj(): pass\n",
    "\n",
    "payload_basic = Obj()\n",
    "payload_basic.dataSchema = Obj()\n",
    "payload_basic.dataSchema.fieldNames = ['responseid', 'weight']\n",
    "payload_basic.data = responseweights_basic.to_dict(orient='records')\n",
    "payload_basic_json = json.dumps(payload_basic, default=lambda o: o.__dict__)\n",
    "\n",
    "payload_adv = Obj()\n",
    "payload_adv.dataSchema = Obj()\n",
    "payload_adv.dataSchema.fieldNames = ['responseid', 'weight']\n",
    "payload_adv.data = responseweights_adv.to_dict(orient='records')\n",
    "payload_adv_json = json.dumps(payload_adv, default=lambda o: o.__dict__)\n",
    "\n",
    "payload_cell = Obj()\n",
    "payload_cell.dataSchema = Obj()\n",
    "payload_cell.dataSchema.fieldNames = ['fiscal_year', 'gender', 'age_group', 'region', 'weight']\n",
    "payload_cell.data = wdfdistinct.to_dict(orient='records')\n",
    "payload_cell_json = json.dumps(payload_cell, default=lambda o: o.__dict__)\n",
    "\n",
    "payload_lock = Obj()\n",
    "payload_lock.dataSchema = Obj()\n",
    "payload_lock.dataSchema.fieldNames = ['age_group', 'gender', 'weight']\n",
    "payload_lock.data = interlockedweights.to_dict(orient='records')\n",
    "payload_lock_json = json.dumps(payload_lock, default=lambda o: o.__dict__)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Insert records for response weights in custom table in hub\n",
    "headers = {'Authorization': 'Bearer ' + token, \"Accept\": \"application/json\", \"Content-Type\": \"application/json\"} \n",
    "\n",
    "# Basic weights\n",
    "url = site_url + 'v1/hubs/' + hubid +'/customdata/tables/'+ hub_tableid_basic +'/records'\n",
    "response = req.put(url, data = payload_basic, headers = headers) \n",
    "response.raise_for_status()\n",
    "\n",
    "# Advanced weights\n",
    "url = site_url + 'v1/hubs/' + hubid +'/customdata/tables/'+ hub_tableid_advanced +'/records'\n",
    "response = req.put(url, data = payload_adv_json, headers = headers) \n",
    "response.raise_for_status()\n",
    "\n",
    "# cell weights\n",
    "url = site_url + 'v1/hubs/' + hubid +'/customdata/tables/'+ hub_tableid_cellweights +'/records'\n",
    "response = req.put(url, data = payload_cell_json, headers = headers) \n",
    "response.raise_for_status()\n",
    "\n",
    "# interlocked weights\n",
    "url = site_url + 'v1/hubs/' + hubid +'/customdata/tables/'+ hub_tableid_interlocked +'/records'\n",
    "response = req.put(url, data = payload_lock_json, headers = headers) \n",
    "response.raise_for_status()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Insert weights in survey data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Obj(): pass\n",
    "\n",
    "#Rename column to map survey data variable\n",
    "weights = responseweights_adv.rename(columns={\"weight\": \"adv_weight\"})\n",
    "\n",
    "#Create json for survey weights\n",
    "payload = Obj()\n",
    "payload.dataSchema = Obj()\n",
    "payload.dataSchema.keys = [\"responseid\"]\n",
    "payload.dataSchema.variables = [\"adv_weight\"]\n",
    "payload.data = weights.to_dict(orient='records')\n",
    "\n",
    "payload_json = json.dumps(payload, default=lambda o: o.__dict__)\n",
    "payload_json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get access token\n",
    "response = req.post(idp_url + 'identity/connect/token', \n",
    "                    data = \"grant_type=api-user&scope=pub.surveys pub.hubs\", \n",
    "                    auth=(client_id, client_secret), \n",
    "                    headers = {'Content-Type': 'application/x-www-form-urlencoded'}) \n",
    "token = response.json()['access_token']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {'Authorization': 'Bearer ' + token, \"Accept\": \"application/json\", \"Content-Type\": \"application/json\"} \n",
    "\n",
    "# Advanced weights\n",
    "response = req.put(site_url + 'v1/surveys/' + projectid + '/responses/data', data = payload_json, headers = headers) \n",
    "print(response.status_code)\n",
    "print(response.content)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}