with tutorial the español applications python r session scrapy httr

python - tutorial - the django project



Consulta multi POST(modo de sesión) (1)

utilizando python requests.Session objeto con los siguientes datos llega a la página de resultados:

form1 = {"energy_category": "electricity", "location": "home", "location-home": "shift", "distributor": "7", "postcode": "3000", "energy_concession": "0", "solar": "0", "disclaimer_chkbox": "disclaimer_selected", } form2 = {"person-count":"1", "room-count":"4", "refrigerator-count":"0", "gas-type":"3", "pool-heating":"0", "spaceheating[]":"none", "spacecooling[]":"none", "cloth-dryer":"0", "waterheating[]":"other"} sub_url = "https://compare.switchon.vic.gov.au/submit" with requests.Session() as s: s.post(sub_url, data=form1) r = (s.get("https://compare.switchon.vic.gov.au/energy_questionnaire")) s.post("https://compare.switchon.vic.gov.au/energy_questionnaire/submit", data=form2) r = s.get("https://compare.switchon.vic.gov.au/offers") print(r.content)

Debería ver la h1 coincidente en el html devuelto que ve en la página:

<h1>Your electricity offers</h1>

O usando solicitudes de formularios de scrapy:

import scrapy class Spider(scrapy.Spider): name = ''comp'' start_urls = [''https://compare.switchon.vic.gov.au/energy_questionnaire/submit''] form1 = {"energy_category": "electricity", "location": "home", "location-home": "shift", "distributor": "7", "postcode": "3000", "energy_concession": "0", "solar": "0", "disclaimer_chkbox": "disclaimer_selected", } sub_url = "https://compare.switchon.vic.gov.au/submit" form2 = {"person-count":"1", "room-count":"4", "refrigerator-count":"0", "gas-type":"3", "pool-heating":"0", "spaceheating[]":"none", "spacecooling[]":"none", "cloth-dryer":"0", "waterheating[]":"other"} def start_requests(self): return [scrapy.FormRequest( self.sub_url, formdata=form1, callback=self.parse )] def parse(self, response): return scrapy.FormRequest.from_response( response, formdata=form2, callback=self.after ) def after(self, response): print("<h1>Your electricity offers</h1>" in response.body)

Lo cual podemos verificar tiene el "<h1>Your electricity offers</h1>" :

2016-03-07 12:27:31 [scrapy] DEBUG: Crawled (200) <GET https://compare.switchon.vic.gov.au/offers#list/electricity> (referer: https://compare.switchon.vic.gov.au/energy_questionnaire) True 2016-03-07 12:27:31 [scrapy] INFO: Closing spider (finished)

El siguiente problema es que los datos reales se procesen de forma dinámica, lo que puede verificar si mira el origen de la página de resultados, puede obtener todo el proveedor en formato json:

with requests.Session() as s: s.post(sub_url, data=form1) r = (s.get("https://compare.switchon.vic.gov.au/energy_questionnaire")) s.post("https://compare.switchon.vic.gov.au/energy_questionnaire/submit", data=form2) r = s.get("https://compare.switchon.vic.gov.au/service/offers") print(r.json())

Un fragmento de lo cual es:

{u''pageMetaData'': {u''showDual'': False, u''isGas'': False, u''showTouToggle'': True, u''isElectricityInitial'': True, u''showLoopback'': False, u''isElectricity'': True}, u''offersList'': [{u''offerDetails'': [{u''coolingOffPeriod'': 0, u''retailerUrl'': u''www.peopleenergy.com.au'', u''offerId'': u''PEO33707SR'', u''contractLengthCount'': 1, u''exitFee'': [0], u''hasIncentive'': False, u''tariffDetails'': {}, u''greenpowerAmount'': 0, u''isDirectDebitOnly'': False, u''basePrice'': 1410, u''offerType'': u''Standing offer'', u''offerName'': u''Residential 5-Day Time of Use'', u''conditionalPrice'': 1410, u''fullDiscountedPrice'': 1390, u''greenPower'': 0, u''retailerName'': u''People Energy'', u''intrinsicGreenpowerPercentage'': u''0.0000'', u''contractLength'': [u''None''], u''hasPayOnTimeDiscount'': False, u''greenpowerChargeType'': None, u''tariffType'': u''Time of use'', u''retailerPhone'': u''1300 788 970'', u''isPartDual'': False, u''retailerId'': u''7322'', u''isTouOffer'': True, u''solarType'': None, u''estimatedSolarCredit'': 0, u''offerKey'': u''1645'', u''exitFeeCount'': 0, u''timeDefinition'': u''Local time'', u''retailerImageUrl'': u''img/retailers/big/peopleenergy.png''}], u''isClosed'': False, u''isChecked'': False, u''offerFuelType'': 0}, {u''offerDetails'': [{u''coolingOffPeriod'': 0, u''retailerUrl'': u''www.peopleenergy.com.au'', u''offerId'': u''PEO33773SR'', u''contractLengthCount'': 1, u''exitFee'': [0], u''hasIncentive'': False, u''tariffDetails'': {}, u''greenpowerAmount'': 0, u''isDirectDebitOnly'': False, u''basePrice'': 1500, u''offerType'': u''Standing offer'', u''offerName'': u''Residential Peak Anytime'', u''conditionalPrice'': 1500, u''fullDiscountedPrice'': 1480, u''greenPower'': 0, u''retailerName'': u''People Energy'', u''intrinsicGreenpowerPercentage'': u''0.0000'', u''contractLength'': [u''None''], u''hasPayOnTimeDiscount'': False, u''greenpowerChargeType'': None, u''tariffType'': u''Single rate'', u''retailerPhone'': u''1300 788 970'', u''isPartDual'': False, u''retailerId'': u''7322'', u''isTouOffer'': False, u''solarType'': None, u''estimatedSolarCredit'': 0, u''offerKey'': u''1649'', u''exitFeeCount'': 0, u''timeDefinition'': u''AEST only'', u''retailerImageUrl'': u''img/retailers/big/peopleenergy.png''}], u''isClosed'': False, u''isChecked'': False, u''offerFuelType'': 0}, {u''offerDetails'': [{u''coolingOffPeriod'': 0, u''retailerUrl'': u''www.energythatcould.com.au'', u''offerId'': u''PAC33683SR'', u''contractLengthCount'': 1, u''exitFee'': [0], u''hasIncentive'': False, u''tariffDetails'': {}, u''greenpowerAmount'': 0, u''isDirectDebitOnly'': False, u''basePrice'': 1400, u''offerType'': u''Standing offer'', u''offerName'': u''Vic Home Flex'', u''conditionalPrice'': 1400, u''fullDiscountedPrice'': 1400, u''greenPower'': 0, u''retailerName'': u''Pacific Hydro Retail Pty Ltd'', u''intrinsicGreenpowerPercentage'': u''0.0000'', u''contractLength'': [u''None''], u''hasPayOnTimeDiscount'': False, u''greenpowerChargeType'': None, u''tariffType'': u''Flexible Pricing'', u''retailerPhone'': u''1800 010 648'', u''isPartDual'': False, u''retailerId'': u''15902'', u''isTouOffer'': False, u''solarType'': None, u''estimatedSolarCredit'': 0, u''offerKey'': u''1666'', u''exitFeeCount'': 0, u''timeDefinition'': u''Local time'', u''retailerImageUrl'': u''img/retailers/big/pachydro.jpg''}], u''isClosed'': False, u''isChecked'': False, u''offerFuelType'': 0}, {u''offerDetails'': [{u''coolingOffPeriod'': 0, u''retailerUrl'': u''www.energythatcould.com.au'', u''offerId'': u''PAC33679SR'', u''contractLengthCount'': 1, u''exitFee'': [0], u''hasIncentive'': False, u''tariffDetails'': {}, u''greenpowerAmount'': 0, u''isDirectDebitOnly'': False, u''basePrice'': 1340, u''offerType'': u''Standing offer'', u''offerName'': u''Vic Home Flex'', u''conditionalPrice'': 1340, u''fullDiscountedPrice'': 1340, u''greenPower'': 0, u''retailerName'': u''Pacific Hydro Retail Pty Ltd'', u''intrinsicGreenpowerPercentage'': u''0.0000'', u''contractLength'': [u''None''], u''hasPayOnTimeDiscount'': False, u''greenpowerChargeType'': None, u''tariffType'': u''Single rate'', u''retailerPhone'': u''1800 010 648'', u''isPartDual'': False, u''retailerId'': u''15902'', u''isTouOffer'': False, u''solarType'': None, u''estimatedSolarCredit'': 0, u''offerKey'': u''1680'', u''exitFeeCount'': 0, u''timeDefinition'': u''Local time'', u''retailerImageUrl'': u''img/retailers/big/pachydro.jpg''}], u''isClosed'': False, u''isChecked'': False, u''offerFuelType'': 0}, {u''offerDetails'': [{u''coolingOffPeriod'': 10, u''retailerUrl'': u''www.commander.com'', u''offerId'': u''M2E30367MR'', u''contractLengthCount'': 1, u''exitFee'': [0], u''hasIncentive'': True, u''tariffDetails'': {}, u''greenpowerAmount'': 0, u''isDirectDebitOnly'': False, u''basePrice'': 1370, u''offerType'': u''Market offer'', u''offerName'': u''Citipower Commander Residential Market Offer (CE3CPR-MAT1 + PF1/TF1/GF1)'', u''conditionalPrice'': 1370, u''fullDiscountedPrice'': 1160, u''greenPower'': 0, u''retailerName'': u''Commander Power & Gas'', u''intrinsicGreenpowerPercentage'': u''0.0000'', u''contractLength'': [u''None''], u''hasPayOnTimeDiscount'': True, u''greenpowerChargeType'': None, u''tariffType'': u''Single rate'', u''retailerPhone'': u''13 39 14'', u''isPartDual'': False, u''retailerId'': u''13667'', u''isTouOffer'': False, u''solarType'': None, u''estimatedSolarCredit'': 0, u''offerKey'': u''2384'', u''exitFeeCount'': 0, u''timeDefinition'': u''AEST only'', u''retailerImageUrl'': u''img/retailers/big/commanderpowergas.png''}], u''isClosed'': False, u''isChecked'': False, u''offerFuelType'': 0}, {u''offerDetails'': [{u''coolingOffPeriod'': 10, u''retailerUrl'': u''www.commander.com'', u''offerId'': u''M2E30359MR'', u''contractLengthCount'': 1, u''exitFee'': [0], u''hasIncentive'': True, u''tariffDetails'': {}, u''greenpowerAmount'': 0, u''isDirectDebitOnly'': False, u''basePrice'': 1330, u''offerType'': u''Market offer'', u''offerName'': u''Citipower Commander Residential Market Offer (Flexible Pricing (Peak, Shoulder and Off Peak) (CE3CPR-MCFP1 + PF1/TF1/GF1)'', u''conditionalPrice'': 1330, u''fullDiscountedPrice'': 1140, u''greenPower'': 0, u''retailerName'': u''Commander Power & Gas'', u''intrinsicGreenpowerPercentage'': u''0.0000'', u''contractLength'': [u''None''], u''hasPayOnTimeDiscount'': True, u''greenpowerChargeType'': None, u''tariffType'': u''Time of use'', u''retailerPhone'': u''13 39 14'', u''isPartDual'': False, u''retailerId'': u''13667'', u''isTouOffer'': True, u''solarType'': None, u''estimatedSolarCredit'': 0, u''offerKey'': u''2386'', u''exitFeeCount'': 0, u''timeDefinition'': u''AEST only'', u''retailerImageUrl'': u''img/retailers/big/commanderpowergas.png''}], u''isClosed'': False, u''isChecked'': False, u''offerFuelType'': 0}, {u''offerDetails'': [{u''coolingOffPeriod'': 10, u''retailerUrl'': u''www.commander.com'', u''offerId'': u''M2E33241MR'', u''contractLengthCount'': 1, u''exitFee'': [0], u''hasIncentive'': True, u''tariffDetails'': {}, u''greenpowerAmount'': 0, u''isDirectDebitOnly'': False, u''basePrice'': 1300, u''offerType'': u''Market offer'', u''offerName'': u''Citipower Commander Residential Market Offer (Peak / Off Peak) (CE3CPR-MPK1OP1)'', u''conditionalPrice'': 1300, u''fullDiscountedPrice'': 1100, u''greenPower'': 0, u''retailerName'': u''Commander Power & Gas'', u''intrinsicGreenpowerPercentage'': u''0.0000'', u''contractLength'': [u''None''], u''hasPayOnTimeDiscount'': True, u''greenpowerChargeType'': None, u''tariffType'': u''Time of use'', u''retailerPhone'': u''13 39 14'', u''isPartDual'': False, u''retailerId'': u''13667'', u''isTouOffer'': True, u''solarType'': None, u''estimatedSolarCredit'': 0, u''offerKey'': u''2389'', u''exitFeeCount'': 0, u''timeDefinition'': u''AEST only'', u''retailerImageUrl'': u''img/retailers/big/commanderpowergas.png''}], u''isClosed'': False, u''isChecked'': False, u''offerFuelType'': 0}, {u''offerDetails'': [{u''coolingOffPeriod'': 0, u''retailerUrl'': u''www.commander.com'', u''offerId'': u''M2E30379SR'', u''contractLengthCount'': 1, u''exitFee'': [0], u''hasIncentive'': False, u''tariffDetails'': {}, u''greenpowerAmount'': 0, u''isDirectDebitOnly'': False, u''basePrice'': 1370, u''offerType'': u''Standing offer'', u''offerName'': u''Citipower Commander Residential Standing Offer (CE3CPR-SAT1 + PF1/TF1/GF1)'', u''conditionalPrice'': 1370, u''fullDiscountedPrice'': 1370, u''greenPower'': 0, u''retailerName'': u''Commander Power & Gas'', u''intrinsicGreenpowerPercentage'': u''0.0000'', u''contractLength'': [u''None''], u''hasPayOnTimeDiscount'': False, u''greenpowerChargeType'': None, u''tariffType'': u''Single rate'', u''retailerPhone'': u''13 39 14'', u''isPartDual'': False, u''retailerId'': u''13667'', u''isTouOffer'': False, u''solarType'': None, u''estimatedSolarCredit'': 0, u''offerKey'': u''2391'', u''exitFeeCount'': 0, u''timeDefinition'': u''AEST only'', u''retailerImageUrl'': u''img/retailers/big/commanderpowergas.png''}], u''isClosed'': False, u''isChecked'': False, u''offerFuelType'': 0}, {u''offerDetails'': [{u''coolingOffPeriod'': 0, u''retailerUrl'': u''www.commander.com'', u''offerId'': u''M2E30369SR'', u''contractLengthCount'': 1, u''exitFee'': [0], u''hasIncentive'': False, u''tariffDetails'': {}, u''greenpowerAmount'': 0, u''isDirectDebitOnly'': False, u''basePrice'': 1330, u''offerType'': u''Standing offer'', u''offerName'': u''Citipower Commander Residential Standing Offer (Flexible Pricing (Peak, Shoulder and Off Peak) (CE3CPR-SCFP1 + PF1/TF1/GF1)'', u''conditionalPrice'': 1330, u''fullDiscountedPrice'': 1330, u''greenPower'': 0, u''retailerName'': u''Commander Power & Gas'', u''intrinsicGreenpowerPercentage'': u''0.0000'', u''contractLength'': [u''None''], u''hasPayOnTimeDiscount'': False, u''greenpowerChargeType'': None, u''tariffType'': u''Time of use'', u''retailerPhone'': u''13 39 14'', u''isPartDual'': False, u''retailerId'': u''13667'', u''isTouOffer'': True, u''solarType'': None, u''estimatedSolarCredit'': 0, u''offerKey'': u''2393'', u''exitFeeCount'': 0, u''timeDefinition'': u''AEST only'', u''retailerImageUrl'': u''img/retailers/big/commanderpowergas.png''}], u''isClosed'': False, u''isChecked'': False, u''offerFuelType'': 0}, {u''offerDetails'': [{u''coolingOffPeriod'': 0, u''retailerUrl'': u''www.commander.com'', u''offerId'': u''M2E30375SR'', u''contractLengthCount'': 1, u''exitFee'': [0], u''hasIncentive'': False, u''tariffDetails'': {}, u''greenpowerAmount'': 0, u''isDirectDebitOnly'': False, u''basePrice'': 1300, u''offerType'': u''Standing offer'', u''offerName'': u''Citipower Commander Residential Standing Offer (Peak / Off Peak) (CE3CPR-SPK1OP1)'', u''conditionalPrice'': 1300, u''fullDiscountedPrice'': 1300, u''greenPower'': 0, u''retailerName'': u''Commander Power & Gas'', u''intrinsicGreenpowerPercentage'': u''0.0000'', u''contractLength'': [u''None''], u''hasPayOnTimeDiscount'': False, u''greenpowerChargeType'': None, u''tariffType'': u''Time of use'', u''retailerPhone'': u''13 39 14'', u''isPartDual'': False, u''retailerId'': u''13667'', u''isTouOffer'': True, u''solarType'': None, u''estimatedSolarCredit'': 0, u''offerKey'': u''2395'', u''exitFeeCount'': 0, u''timeDefinition'': u''AEST only'', u''retailerImageUrl'': u''img/retailers/big/commanderpowergas.png''}], u''isClosed'': False, u''isChecked'': False, u''offerFuelType'': 0}, {u''offerDetails'': [{u''coolingOffPeriod'': 0, u''retailerUrl'': u''www.dodo.com/powerandgas'', u''offerId'': u''DOD32903SR'', u''contractLengthCount'': 1, u''exitFee'': [0], u''hasIncentive'': False, u''tariffDetails'': {}, u''greenpowerAmount'': 0, u''isDirectDebitOnly'': False, u''basePrice'': 1320, u''offerType'': u''Standing offer'', u''offerName'': u''Citipower Res No Term Standing Offer (Common Form Flex Plan) (E3CPR-SCFP1)'', u''conditionalPrice'': 1320, u''fullDiscountedPrice'': 1320, u''greenPower'': 0, u''retailerName'': u''Dodo Power & Gas'', u''intrinsicGreenpowerPercentage'': u''0.0000'', u''contractLength'': [u''None''], u''hasPayOnTimeDiscount'': False, u''greenpowerChargeType'': None, u''tariffType'':

Luego, si mira las solicitudes más tarde, por ejemplo, cuando hace clic en el botón Comparar seleccionado en la página de resultados, hay una solicitud como:

https://compare.switchon.vic.gov.au/service/offer/tariff/9090/9092

Entonces, puede imitar lo que sucede filtrando usando la tarifa o alguna variación.

En realidad, puede obtener todos los datos como json, si ingresa los mismos valores que figuran a continuación en los formularios:

form1 = {"energy_category": "electricity", "location": "home", "location-home": "shift", "distributor": "7", "postcode": "3000", "energy_concession": "0", "solar": "0", "disclaimer_chkbox": "disclaimer_selected" } form2 = {"person-count":"1", "room-count":"1", "refrigerator-count":"1", "gas-type":"4", "pool-heating":"0", "spaceheating[]":"none", "spacecooling[]":"none", "cloth-dryer":"0", "cloth-dryer-freq-weekday":"", "waterheating[]":"other"} import json with requests.Session() as s: s.post(sub_url, data=form1) r = (s.get("https://compare.switchon.vic.gov.au/energy_questionnaire")) s.post("https://compare.switchon.vic.gov.au/energy_questionnaire/submit", data=form2) js = s.get("https://compare.switchon.vic.gov.au/service/offers").json()["offersList"] by_discount = sorted(js, key=lambda d: d["offerDetails"][0]["fullDiscountedPrice"])

Si simplemente extraemos los dos primeros valores de la lista ordenada por el precio de descuento total:

from pprint import pprint as pp pp(by_discount[:2])

Conseguirás:

[{u''isChecked'': False, u''isClosed'': False, u''offerDetails'': [{u''basePrice'': 980, u''conditionalPrice'': 980, u''contractLength'': [u''None''], u''contractLengthCount'': 1, u''coolingOffPeriod'': 10, u''estimatedSolarCredit'': 0, u''exitFee'': [0], u''exitFeeCount'': 1, u''fullDiscountedPrice'': 660, u''greenPower'': 0, u''greenpowerAmount'': 0, u''greenpowerChargeType'': None, u''hasIncentive'': False, u''hasPayOnTimeDiscount'': True, u''intrinsicGreenpowerPercentage'': u''0.0000'', u''isDirectDebitOnly'': False, u''isPartDual'': False, u''isTouOffer'': False, u''offerId'': u''GLO40961MR'', u''offerKey'': u''7636'', u''offerName'': u''GLO SWITCH'', u''offerType'': u''Market offer'', u''retailerId'': u''31206'', u''retailerImageUrl'': u''img/retailers/big/globird.jpg'', u''retailerName'': u''GloBird Energy'', u''retailerPhone'': u''(03) 8560 4199'', u''retailerUrl'': u''http://www.globirdenergy.com.au/switchon/'', u''solarType'': None, u''tariffDetails'': {}, u''tariffType'': u''Single rate'', u''timeDefinition'': u''Local time''}], u''offerFuelType'': 0}, {u''isChecked'': False, u''isClosed'': False, u''offerDetails'': [{u''basePrice'': 1080, u''conditionalPrice'': 1080, u''contractLength'': [u''None''], u''contractLengthCount'': 1, u''coolingOffPeriod'': 10, u''estimatedSolarCredit'': 0, u''exitFee'': [0], u''exitFeeCount'': 1, u''fullDiscountedPrice'': 720, u''greenPower'': 0, u''greenpowerAmount'': 0, u''greenpowerChargeType'': None, u''hasIncentive'': False, u''hasPayOnTimeDiscount'': True, u''intrinsicGreenpowerPercentage'': u''0.0000'', u''isDirectDebitOnly'': False, u''isPartDual'': False, u''isTouOffer'': True, u''offerId'': u''GLO41009MR'', u''offerKey'': u''7642'', u''offerName'': u''GLO SWITCH'', u''offerType'': u''Market offer'', u''retailerId'': u''31206'', u''retailerImageUrl'': u''img/retailers/big/globird.jpg'', u''retailerName'': u''GloBird Energy'', u''retailerPhone'': u''(03) 8560 4199'', u''retailerUrl'': u''http://www.globirdenergy.com.au/switchon/'', u''solarType'': None, u''tariffDetails'': {}, u''tariffType'': u''Time of use'', u''timeDefinition'': u''Local time''}], u''offerFuelType'': 0}]

Que debe coincidir con lo que ve en la página cuando hace clic en el botón de filtro "DISCOUNTED PRICE" .

Para la vista normal, parece que está ordenada por conditionalPrice o basePrice , y una vez más, los dos primeros valores deben coincidir con los que ve en la página web:

base = sorted(js, key=lambda d: d["offerDetails"][0]["conditionalPrice"]) from pprint import pprint as pp pp(base[:2]) [{u''isChecked'': False, u''isClosed'': False, u''offerDetails'': [{u''basePrice'': 740, u''conditionalPrice'': 740, u''contractLength'': [u''None''], u''contractLengthCount'': 1, u''coolingOffPeriod'': 0, u''estimatedSolarCredit'': 0, u''exitFee'': [0], u''exitFeeCount'': 0, u''fullDiscountedPrice'': 740, u''greenPower'': 0, u''greenpowerAmount'': 0, u''greenpowerChargeType'': None, u''hasIncentive'': False, u''hasPayOnTimeDiscount'': False, u''intrinsicGreenpowerPercentage'': u''0.0000'', u''isDirectDebitOnly'': False, u''isPartDual'': False, u''isTouOffer'': False, u''offerId'': u''NEX42694SR'', u''offerKey'': u''9092'', u''offerName'': u''Citpower Single Rate Residential'', u''offerType'': u''Standing offer'', u''retailerId'': u''35726'', u''retailerImageUrl'': u''img/retailers/big/nextbusinessenergy.jpg'', u''retailerName'': u''Next Business Energy Pty Ltd'', u''retailerPhone'': u''1300 466 398'', u''retailerUrl'': u''http://www.nextbusinessenergy.com.au/'', u''solarType'': None, u''tariffDetails'': {}, u''tariffType'': u''Single rate'', u''timeDefinition'': u''Local time''}], u''offerFuelType'': 0}, {u''isChecked'': False, u''isClosed'': False, u''offerDetails'': [{u''basePrice'': 780, u''conditionalPrice'': 780, u''contractLength'': [u''None''], u''contractLengthCount'': 1, u''coolingOffPeriod'': 0, u''estimatedSolarCredit'': 0, u''exitFee'': [0], u''exitFeeCount'': 0, u''fullDiscountedPrice'': 780, u''greenPower'': 0, u''greenpowerAmount'': 0, u''greenpowerChargeType'': None, u''hasIncentive'': False, u''hasPayOnTimeDiscount'': False, u''intrinsicGreenpowerPercentage'': u''0.0000'', u''isDirectDebitOnly'': False, u''isPartDual'': False, u''isTouOffer'': False, u''offerId'': u''NEX42699SR'', u''offerKey'': u''9090'', u''offerName'': u''Citpower Residential Flexible Pricing'', u''offerType'': u''Standing offer'', u''retailerId'': u''35726'', u''retailerImageUrl'': u''img/retailers/big/nextbusinessenergy.jpg'', u''retailerName'': u''Next Business Energy Pty Ltd'', u''retailerPhone'': u''1300 466 398'', u''retailerUrl'': u''http://www.nextbusinessenergy.com.au/'', u''solarType'': None, u''tariffDetails'': {}, u''tariffType'': u''Flexible Pricing'', u''timeDefinition'': u''Local time''}], u''offerFuelType'': 0}]

Puede ver todo el json devuelto en la consola Firebug si hace clic en https://compare.switchon.vic.gov.au/service/offers get entry y luego pulsa la respuesta :

Debería poder sacar cada campo que desee de eso.

El resultado en realidad tiene algunos resultados adicionales que no se ven en la página a menos que se active el siguiente botón tou:

Puede filtrar los resultados para que coincida exactamente con la salida predeterminada o para incluir una opción con una función auxiliar:

def order_by(l, k, is_tou=False): if not is_tou: filt = filter(lambda x: not x["offerDetails"][0]["isTouOffer"], l) return sorted(filt, key=lambda d: d["offerDetails"][0][k]) return sorted(l, key=lambda d: d["offerDetails"][0][k]) import json with requests.Session() as s: s.post(sub_url, data=form1) r = (s.get("https://compare.switchon.vic.gov.au/energy_questionnaire")) s.post("https://compare.switchon.vic.gov.au/energy_questionnaire/submit", data=form2) js = s.get("https://compare.switchon.vic.gov.au/service/offers").json()["offersList"] by_price = by_discount(js, "conditionalPrice", False) print(by_price[:3)

Si comprueba el resultado, verá el tercer origen de energía con un precio de 840 en los resultados con el interruptor encendido o 860 para AGL cuando está apagado, puede aplicar lo mismo al resultado de descuento:

El resultado regular también parece estar ordenado por conditionalPrice si verifica la fuente. Las dos funciones js que se solicitan para ordenar son:

ng-click="changeSortingField(''conditionalPrice'')" ng-click="changeSortingField(''fullDiscountedPrice'')"

Entonces eso definitivamente definitivamente debería coincidir completamente con la salida del sitio.

Estoy tratando de interrogar a este sitio para obtener la lista de ofertas. El problema es que necesitamos completar 2 formularios (2 consultas POST) antes de recibir el resultado final.

Esto que he hecho hasta ahora:

Primero envío el primer POST después de configurar las cookies:

library(httr) set_cookies(.cookies = c(a = "1", b = "2")) first_url <- "https://compare.switchon.vic.gov.au/submit" body <- list(energy_category="electricity", location="home", "location-home"="shift", "retailer-company"="", postcode="3000", distributor=7, zone=1, energy_concession=0, "file-provider"="", solar=0, solar_feedin_tariff="", disclaimer_chkbox="disclaimer_selected") qr<- POST(first_url, encode="form", body=body)

Luego, intente recuperar las ofertas utilizando la segunda consulta de publicación:

gov_url <- "https://compare.switchon.vic.gov.au/energy_questionnaire/submit" qr1<- POST(gov_url, encode="form", body=list(`person-count`=1, `room-count`=1, `refrigerator-count`=1, `gas-type`=4, `pool-heating`=0, spaceheating="none", spacecooling="none", `cloth-dryer`=0, waterheating="other"), set_cookies(a = 1, b = 2)) ) library(XML) dc <- htmlParse(qr1)

Pero desafortunadamente recibo un mensaje que indica el final de la sesión. Muchas gracias por cualquier ayuda para resolver esto.

actualizar agregar cookies:

Agregué las cookies y el GET intermedio, pero todavía no tengo ninguno de los resultados.

library(httr) first_url <- "https://compare.switchon.vic.gov.au/submit" body <- list(energy_category="electricity", location="home", "location-home"="shift", "retailer-company"="", postcode=3000, distributor=7, zone=1, energy_concession=0, "file-provider"="", solar=0, solar_feedin_tariff="", disclaimer_chkbox="disclaimer_selected") qr<- POST(first_url, encode="form", body=body, config=set_cookies(a = 1, b = 2)) xx <- GET("https://compare.switchon.vic.gov.au/energy_questionnaire",config=set_cookies(a = 1, b = 2)) gov_url <- "https://compare.switchon.vic.gov.au/energy_questionnaire/submit" qr1<- POST(gov_url, encode="form", body=list( `person-count`=1, `room-count`=1, `refrigerator-count`=1, `gas-type`=4, `pool-heating`=0, spaceheating="none", spacecooling="none", `cloth-dryer`=0, waterheating="other"), config=set_cookies(a = 1, b = 2)) library(XML) dc <- htmlParse(qr1)