diff --git a/.changeset/giant-years-help.md b/.changeset/giant-years-help.md new file mode 100644 index 0000000000..c6ca0def30 --- /dev/null +++ b/.changeset/giant-years-help.md @@ -0,0 +1,7 @@ +--- +'@backstage/backend-common': patch +'@backstage/cli': patch +'@backstage/config-loader': patch +--- + +Reading app config from a remote server diff --git a/ADOPTERS.md b/ADOPTERS.md index 64bd99e62f..06501a2a10 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -1,65 +1,65 @@ -| Organization | Contact | Description of Use | -| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Spotify](https://www.spotify.com) | [@leemills83](https://github.com/leemills83) | Main interface towards all of Spotify's infrastructure and technical documentation. | -| [bol.com](https://www.bol.com) | [@sagacity](https://github.com/sagacity) | Initial work being done to unify platform tooling. | -| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | -| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | -| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | -| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | -| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | -| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | -| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | -| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | -| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | -| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | -| [Telenor Sweden](https://www.telenor.se) | [@O5ten](https://github.com/O5ten) | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks | -| [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | -| [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | -| [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | -| [Expedia Group](https://www.expediagroup.com) | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit | -| [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | -| [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | -| [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | -| [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | -| [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | -| [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | -| [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a teamโ€™s engineering dependencies. | -| [Oriflame](https://www.oriflame.com/) | [Oriflame](https://github.com/oriflame) | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more. | -| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH) | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments. | -| [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | -| [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | -| [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | -| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process ๐ŸŒ•๐Ÿš€๐Ÿง‘โ€๐Ÿš€ | -| [FundApps](https://www.fundapps.co/) | [Elliot Greenwood](https://github.com/egnwd) | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes | -| [DAZN](https://dazn.com/) | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com) | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). | -| [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | -| [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | -| [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | -| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | -| [MavTek](https://www.mavtek.com/) | [@fgascon](https://github.com/fgascon) | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices. | -| [QuintoAndar](https://www.quintoandar.com.br/) | [@quintoandar](https://github.com/quintoandar) | Developer portal, services catalog and centralization of service metrics. | -| [empathy.co](https://empathy.co/) | [@guillermotti](https://github.com/guillermotti) | Developer portal for tech docs, service catalog, plugin discovery and much more. | -| [creditas.com](https://creditas.com/) | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas) | Centralization of all services, standards, documentation, etc. We started the deployment process. | -| [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl) | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration | -| [Powerspike](https://powerspike.tv/) | [@trelore](https://github.com/trelore) | Developer portal for documentation of core libraries and repositories. | -| [2U](https://2u.com) | [Andrew Thal](https://github.com/athal7) | Development team home-base, promoting service discoverability, resource dependencies, and tech radar | -| [Taxfix](https://taxfix.de/) | [Sami Ur Rehman](https://github.com/samiurrehman92) | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins. | -| [Busuu](https://busuu.com/) | [Adam Tester](https://github.com/adamtester) | Developer portal with service catalog, API docs, Event docs, service templating, and cost insights. | -| [Loadsmart](https://loadsmart.com/) | [Loadsmart](https://github.com/loadsmart) | Improve services visibility and operations for service owners and developers. | -| [Monzo](https://monzo.com/) | [@WillSewell](https://github.com/WillSewell), [@joechrisellis](https://github.com/joechrisellis) | Developer portal showing metadata and docs for over 2000 microservices. We have built a number of plugins such as a UI for our system to measure [software excellence](https://monzo.com/blog/2021/09/15/how-we-measure-software-excellence), and a UI to show deployment and config change events. | -| [Vaimo](https://www.vaimo.com) | [@vaimo-magnus](https://github.com/vaimo-magnus) | Developer Portal for our developers at Vaimo, currently docs and self-service towards our internal PaaS based on k8s. Plans to extend the catalog into Projects, Environments etc | -| [Wayfair](https://www.wayfair.com) | [@fransan](https://github.com/fransan), [@errskipower](https://github.com/errskipower), [@hrrs](https://github.com/hrrs) | Developer portal for service catalog, technical documentation, and APIs. | -| [CircleHD](https://www.circlehd.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | -| [CastDesk](https://castdesk.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | -| [Santagostino](https://santagostino.it) | [@santagostino](https://github.com/santagostino) | Developer portal, gateway to our infrastructure, documentation, service catalog and internal tooling. | -| [Peak](https://peak.ai) | [Luke Beamish](https://github.com/lukebeamish-peak) | Developer portal for all internal engineers to access documentation and tooling. | -| [Gelato](https://gelato.com/) | [Dmitry Makarenko](https://github.com/dmitry-makarenko-gelato) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal and third-party systems๐Ÿš€. | -| [GoCardless](https://gocardless.com/) | [James Turley](https://github.com/tragiclifestories) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal systems. | -| [Box](https://www.box.com) | [@kielosz](https://github.com/kielosz), [@jluk-box](https://github.com/jluk-box), [@ptychu](https://github.com/ptychu), [@alexrybch](https://github.com/alexrybch), [@szubster](https://github.com/szubster) | Developer portal for service catalog, integration with internal systems, new service onboarding. | -| [Bazaarvoice](https://www.bazaarvoice.com) | [@niallmccullagh](https://github.com/niallmccullagh) | Developer portal for service catalog and scaffolds, publishing Github docs and API documentation, visualising our internal tech radar and our product engineering org structure. | -| [Krateo PlatformOps](https://www.krateo.io) | [@projectkerberus](https://github.com/projectkerberus) | A multi-cloud control plane to create, manage and deploy any kind of resource easily and centrally via a Developer Portal that centralizes via a self-service catalog the templating and ownership of services, the available documentation, the overview of the components that compose an entire domain and all the data of the service lifecycle. | -| [Adevinta](https://www.adevinta.com) | [Ray Sinnema](https://github.com/RemonSinnema) | Showcase shared services to our internal customers. | -| [Splunk](https://www.splunk.com) | [@tonytamsf](https://github.com/tonytamsf) | Developer portal as a centralized place to find people, services, documentation, escalation policies and give bravos. This portal is also being used as a centralized search engine for engineering specific documentation.| -| [SoundCloud](https://www.soundcloud.com) | [Julio Zynger](https://github.com/julioz) | Developer portal as a [humane registry](https://martinfowler.com/bliki/HumaneRegistry.html) for the organization: catalog of people, services, documentation, feature toggles, escalation policies, etc.| -| [Volvofinans Bank](https://www.volvofinans.se) | [Johan Hammar](https://github.com/johanhammar) | Developer portal enabling engineers to manage and explore software and documentation. | \ No newline at end of file +| Organization | Contact | Description of Use | +| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Spotify](https://www.spotify.com) | [@leemills83](https://github.com/leemills83) | Main interface towards all of Spotify's infrastructure and technical documentation. | +| [bol.com](https://www.bol.com) | [@sagacity](https://github.com/sagacity) | Initial work being done to unify platform tooling. | +| [DFDS](https://www.dfds.com) | [@carlsendk](https://github.com/carlsendk) | V2 self-service platform. | +| [Roadie](https://roadie.io) | [@dtuite](https://github.com/dtuite) | Hosted, managed Backstage with easy set-up | +| [Roku](https://www.roku.com) | [@timurista](https://github.com/timurista) | Initial work on Cloud engineering service platform. | +| [SDA SE](https://sda.se) | [@Fox32](https://github.com/Fox32) | Central place for developing and sharing services in our insurance ecosystem. | +| [H-E-B](https://www.heb.com) | [@german-j-rodriguez](https://github.com/german-j-rodriguez) | Initial work on Engineering Portal service platform. | +| [American Airlines](https://www.aa.com) | [@paulpach](https://github.com/paulpach) | Central place for developers to develop and maintain applications | +| [Kiwi.com](https://kiwi.com) | [@aexvir](https://github.com/aexvir) | Replacing the frontend of [The Zoo](https://github.com/kiwicom/the-zoo), their service registry. | +| [Voi](https://www.voiscooters.com/) | [@K-Phoen](https://github.com/K-Phoen) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [Talkdesk](https://www.talkdesk.com) | [@jaime-talkdesk](https://github.com/jaime-talkdesk) | Initial work for Engineering Portal and Self Provisioning to R&D | +| [Wealthsimple](https://www.wealthsimple.com) | [@andrewthauer](https://github.com/andrewthauer) | Developer portal, service catalog, documentation and tooling | +| [Grab](https://www.grab.com) | [@althafh](https://github.com/althafh) | Initial work as a unified interface for all of Grab's internal tooling | +| [Telenor Sweden](https://www.telenor.se) | [@O5ten](https://github.com/O5ten) | Building a developer portal for scaffolding projects towards our unified build environment and microservice stacks | +| [Fiverr](https://www.fiverr.com) | [@nirga](https://github.com/nirga) | Unifying separate tools that developers are using today (i.e. monitoring, dead letter queues management, etc.) into a single platform. | +| [Zalando SE](https://www.zalando.de) | [@leviferreira](https://github.com/leviferreira) | Building V2 of the Internal Development Portal. | +| [LegalZoom](https://legalzoom.com) | [@backjo](https://github.com/backjo) | Developer portal - hub for all engineering projects and metadata. | +| [Expedia Group](https://www.expediagroup.com) | [Mike Turner](mailto:miturner@expediagroup.com), [Sneha Kumar](mailto:snkumar@expediagroup.com), [@guillermomanzo](https://github.com/guillermomanzo), [Erik Lindgren](https://github.com/lindgren) | EG Common Developer Toolkit | +| [Paddle.com](https://paddle.com) | [Ioannis Georgoulas](https://github.com/geototti21) | Developer portal (Tech Docs, Service Catalog, Internal Tooling), we use vanilla Backstage FE and custom BE implementation in Go | +| [Acast.com](https://acast.com) | [Olle Lundberg](https://github.com/lndbrg) | Developer portal with tech docs, service catalog and a bunch of other internal tooling | +| [Lunar](https://lunar.app) | [Jacob Valdemar](https://github.com/JacobValdemar) | Internal developer portal for service overview and insights, API documentation, technical guides, onboarding guides and RFC's. | +| [Trendyol](https://trendyol.com) | [Gamze Senturk](https://github.com/gmzsenturk), [Mert Can Bilgic](https://github.com/mertcb) | The Developer Portal has been called `Pandora`. Provides an overview of Trendyol tech ecosystem. TechDocs, Catalog, Custom Plugins and Theme. | +| [Peloton](https://www.onepeloton.com/) | [Jim Haughwout](https://github.com/JimHaughwout) | Creating our first developer portal and tech-docs. Exploring Service Catalog, Tech Insights and Cost Insights as well. | +| [TELUS](https://telus.com) | [Seb Barre](https://github.com/sbarre) | The Go-to place to find answers about development and delivery at TELUS. | +| [Brex](https://www.brex.com/) | [Vamsi Chitters](https://github.com/vamsikc) | A centralized UI to understand how a service fits in the whole Brex architecture and manage a teamโ€™s engineering dependencies. | +| [Oriflame](https://www.oriflame.com/) | [Oriflame](https://github.com/oriflame) | Internal developer portal for services, single page apps and packages overview, API documentation, technical guides, tech-radar and more. | +| [Booz Allen Hamilton](https://www.boozallen.com/) | [Jason Miller](https://github.com/JasonMiller-BAH) | Developer portal for a full-stack software development ecosystem that accelerates consistent and repeatable Modern Software Development practices for internal innovation and investments. | +| [Netflix](https://www.netflix.com/) | [bleathem](https://github.com/bleathem) | Our Backstage implementation will be the front door to a unified experience connecting our internal platform products across important workflows with integrated knowledge and support. | +| [b.well](https://www.icanbwell.com/) | [Jacob Rosales](https://github.com/jrosales) | Foundation for our engineering portal and cloud insights. | +| [PagerDuty](https://www.pagerduty.com/) | [Mark Shaw](https://github.com/markshawtoronto) | Developer portal, initially focused on software templates and tech-docs. | +| [MoonShiner](https://moonshiner.at) | [Fabian Hippmann](https://github.com/FabianHippmann) | Developer portal - helps us keep track of our customer projects, onboard new developers & improve our development process ๐ŸŒ•๐Ÿš€๐Ÿง‘โ€๐Ÿš€ | +| [FundApps](https://www.fundapps.co/) | [Elliot Greenwood](https://github.com/egnwd) | Developer Portal - A place for us to keep track of our projects and documentation for all services and processes | +| [DAZN](https://dazn.com/) | [Lou Bichard](https://twitter.com/loujaybee), [Marco Crivellaro](https://github.com/crivetechie), [Alex Hollerith](mailto:alex.hollerith@dazn.com) | Ingesting all of DAZN's repos for the catalog, migrating our internal platform apps (pull request boards, release information, inner source marketplace etc) to Backstage plugins (where applicable). | +| [HelloFresh](https://www.hellofresh.de/) | [@iammuho](https://github.com/iammuho), [@ElenaForester](https://github.com/ElenaForester), [@diegomarangoni](https://github.com/diegomarangoni) | Our developer portal at HelloFresh - Spread across an organisation of 500+ engineers globally. | +| [FactSet](https://www.factset.com/) | [@kuangp](https://github.com/kuangp) | Developer portal to provide discoverability to all internal components, APIs, documentation, and scaffold templates with integrations to our internal infrastructure tools. | +| [Workrise](https://www.workrise.com/) | [Michael Rode](https://github.com/michaelrode) | Developer portal, main gateway to our infrastructure, documentation and internal tooling. | +| [RedVentures](https://www.redventures.com/) | [Chris Diaz](https://github.com/codingdiaz) | Developer portal that brings everything an engineer needs to provide value into a single pane of glass. | +| [MavTek](https://www.mavtek.com/) | [@fgascon](https://github.com/fgascon) | Developer portal focused on standardizing practices, centralizing documentation and streamlining developer practices. | +| [QuintoAndar](https://www.quintoandar.com.br/) | [@quintoandar](https://github.com/quintoandar) | Developer portal, services catalog and centralization of service metrics. | +| [empathy.co](https://empathy.co/) | [@guillermotti](https://github.com/guillermotti) | Developer portal for tech docs, service catalog, plugin discovery and much more. | +| [creditas.com](https://creditas.com/) | [@aureliosaraiva](https://github.com/aureliosaraiva) [@Creditas](https://github.com/creditas) | Centralization of all services, standards, documentation, etc. We started the deployment process. | +| [Prisjakt](https://www.prisjakt.nu) / [PriceSpy](https://pricespy.co.uk) | [@kennylindahl](https://github.com/kennylindahl) | Internal developer portal - Documentation, scaffolding, software catalog, TechRadar, Gitlab org data integration | +| [Powerspike](https://powerspike.tv/) | [@trelore](https://github.com/trelore) | Developer portal for documentation of core libraries and repositories. | +| [2U](https://2u.com) | [Andrew Thal](https://github.com/athal7) | Development team home-base, promoting service discoverability, resource dependencies, and tech radar | +| [Taxfix](https://taxfix.de/) | [Sami Ur Rehman](https://github.com/samiurrehman92) | Developer's portal with software catalog at it's core. Hosts API Specs, Tech Docs, Tech Radar and some custom plugins. | +| [Busuu](https://busuu.com/) | [Adam Tester](https://github.com/adamtester) | Developer portal with service catalog, API docs, Event docs, service templating, and cost insights. | +| [Loadsmart](https://loadsmart.com/) | [Loadsmart](https://github.com/loadsmart) | Improve services visibility and operations for service owners and developers. | +| [Monzo](https://monzo.com/) | [@WillSewell](https://github.com/WillSewell), [@joechrisellis](https://github.com/joechrisellis) | Developer portal showing metadata and docs for over 2000 microservices. We have built a number of plugins such as a UI for our system to measure [software excellence](https://monzo.com/blog/2021/09/15/how-we-measure-software-excellence), and a UI to show deployment and config change events. | +| [Vaimo](https://www.vaimo.com) | [@vaimo-magnus](https://github.com/vaimo-magnus) | Developer Portal for our developers at Vaimo, currently docs and self-service towards our internal PaaS based on k8s. Plans to extend the catalog into Projects, Environments etc | +| [Wayfair](https://www.wayfair.com) | [@fransan](https://github.com/fransan), [@errskipower](https://github.com/errskipower), [@hrrs](https://github.com/hrrs) | Developer portal for service catalog, technical documentation, and APIs. | +| [CircleHD](https://www.circlehd.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | +| [CastDesk](https://castdesk.com) | [@circlehddev](https://github.com/circlehddev) | Developer Portal for internal dev team across the globe | +| [Santagostino](https://santagostino.it) | [@santagostino](https://github.com/santagostino) | Developer portal, gateway to our infrastructure, documentation, service catalog and internal tooling. | +| [Peak](https://peak.ai) | [Luke Beamish](https://github.com/lukebeamish-peak) | Developer portal for all internal engineers to access documentation and tooling. | +| [Gelato](https://gelato.com/) | [Dmitry Makarenko](https://github.com/dmitry-makarenko-gelato) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal and third-party systems๐Ÿš€. | +| [GoCardless](https://gocardless.com/) | [James Turley](https://github.com/tragiclifestories) | Developer portal: documentation, service templates, org structure, service catalog, plugins for integration with internal systems. | +| [Box](https://www.box.com) | [@kielosz](https://github.com/kielosz), [@jluk-box](https://github.com/jluk-box), [@ptychu](https://github.com/ptychu), [@alexrybch](https://github.com/alexrybch), [@szubster](https://github.com/szubster) | Developer portal for service catalog, integration with internal systems, new service onboarding. | +| [Bazaarvoice](https://www.bazaarvoice.com) | [@niallmccullagh](https://github.com/niallmccullagh) | Developer portal for service catalog and scaffolds, publishing Github docs and API documentation, visualising our internal tech radar and our product engineering org structure. | +| [Krateo PlatformOps](https://www.krateo.io) | [@projectkerberus](https://github.com/projectkerberus) | A multi-cloud control plane to create, manage and deploy any kind of resource easily and centrally via a Developer Portal that centralizes via a self-service catalog the templating and ownership of services, the available documentation, the overview of the components that compose an entire domain and all the data of the service lifecycle. | +| [Adevinta](https://www.adevinta.com) | [Ray Sinnema](https://github.com/RemonSinnema) | Showcase shared services to our internal customers. | +| [Splunk](https://www.splunk.com) | [@tonytamsf](https://github.com/tonytamsf) | Developer portal as a centralized place to find people, services, documentation, escalation policies and give bravos. This portal is also being used as a centralized search engine for engineering specific documentation. | +| [SoundCloud](https://www.soundcloud.com) | [Julio Zynger](https://github.com/julioz) | Developer portal as a [humane registry](https://martinfowler.com/bliki/HumaneRegistry.html) for the organization: catalog of people, services, documentation, feature toggles, escalation policies, etc. | +| [Volvofinans Bank](https://www.volvofinans.se) | [Johan Hammar](https://github.com/johanhammar) | Developer portal enabling engineers to manage and explore software and documentation. | diff --git a/docs/conf/writing.md b/docs/conf/writing.md index 4288d23b17..a0e29820ed 100644 --- a/docs/conf/writing.md +++ b/docs/conf/writing.md @@ -67,13 +67,15 @@ production build. ## Configuration Files -It is possible to have multiple configuration files, both to support different -environments, but also to define configuration that is local to specific -packages. The configuration files to load are selected using a `--config ` -flag, and it is possible to load any number of files. Paths are relative to the -working directory of the executed process, for example `package/backend`. This -means that to select a config file in the repo root when running the backend, -you would use `--config ../../my-config.yaml`. +It is possible to have multiple configuration files (bundled and/or remote), +both to support different environments, but also to define configuration that is +local to specific packages. The configuration files to load are selected using a +`--config ` flag, and it is possible to load any number of +files. Paths are relative to the working directory of the executed process, for +example `package/backend`. This means that to select a config file in the repo +root when running the backend, you would use `--config ../../my-config.yaml`, +and for config file on a config server you would use +`--config https://some.domain.io/app-config.yaml` If no `config` flags are specified, the default behavior is to load `app-config.yaml` and, if it exists, `app-config.local.yaml` from the repo root. @@ -85,7 +87,7 @@ are NOT loaded. To include them you need to explicitly include them with a flag, for example: ```shell -yarn start --config ../../app-config.yaml --config ../../app-config.staging.yaml +yarn start --config ../../app-config.yaml --config ../../app-config.staging.yaml --config https://some.domain.io/app-config.yaml ``` All loaded configuration files are merged together using the following rules: diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index cfb655cbeb..3fd5e1b44e 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -22,10 +22,13 @@ import { loadConfigSchema, loadConfig, ConfigSchema, + ConfigTarget, } from '@backstage/config-loader'; import { AppConfig, Config, ConfigReader } from '@backstage/config'; import { JsonValue } from '@backstage/types'; +import { isValidUrl } from './urls'; + import { setRootLoggerRedactionList } from './logging/rootLogger'; // Fetch the schema and get all the secrets to pass to the rootLogger for redaction @@ -178,7 +181,10 @@ export async function loadBackendConfig(options: { argv: string[]; }): Promise { const args = parseArgs(options.argv); - const configPaths: string[] = [args.config ?? []].flat(); + + const configTargets: ConfigTarget[] = [args.config ?? []] + .flat() + .map(arg => (isValidUrl(arg) ? { url: arg } : { path: resolvePath(arg) })); /* eslint-disable-next-line no-restricted-syntax */ const paths = findPaths(__dirname); @@ -196,7 +202,8 @@ export async function loadBackendConfig(options: { const config = new ObservableConfigProxy(options.logger); const configs = await loadConfig({ configRoot: paths.targetRoot, - configPaths: configPaths.map(opt => resolvePath(opt)), + configPaths: [], + configTargets: configTargets, watch: { onChange(newConfigs) { options.logger.info( diff --git a/packages/backend-common/src/urls.test.ts b/packages/backend-common/src/urls.test.ts new file mode 100644 index 0000000000..c2a67fb849 --- /dev/null +++ b/packages/backend-common/src/urls.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isValidUrl } from './urls'; + +describe('isValidUrl', () => { + it('should return true for url', () => { + const validUrl = isValidUrl('http://some.valid.url'); + expect(validUrl).toBe(true); + }); + + it('should return false for absolute path', () => { + const validUrl = isValidUrl('/some/absolute/path'); + expect(validUrl).toBe(false); + }); + + it('should return false for relative path', () => { + const validUrl = isValidUrl('../some/relative/path'); + expect(validUrl).toBe(false); + }); +}); diff --git a/packages/backend-common/src/urls.ts b/packages/backend-common/src/urls.ts new file mode 100644 index 0000000000..848cea25d9 --- /dev/null +++ b/packages/backend-common/src/urls.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isValidUrl(url: string): boolean { + try { + // eslint-disable-next-line no-new + new URL(url); + return true; + } catch { + return false; + } +} diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index 3688d60089..80b374039f 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -14,9 +14,14 @@ * limitations under the License. */ -import { loadConfig, loadConfigSchema } from '@backstage/config-loader'; +import { + ConfigTarget, + loadConfig, + loadConfigSchema, +} from '@backstage/config-loader'; import { ConfigReader } from '@backstage/config'; import { paths } from './paths'; +import { isValidUrl } from './urls'; type Options = { args: string[]; @@ -26,7 +31,12 @@ type Options = { }; export async function loadCliConfig(options: Options) { - const configPaths = options.args.map(arg => paths.resolveTarget(arg)); + const configTargets: ConfigTarget[] = []; + options.args.forEach(arg => { + if (!isValidUrl(arg)) { + configTargets.push({ path: paths.resolveTarget(arg) }); + } + }); // Consider all packages in the monorepo when loading in config const { Project } = require('@lerna/project'); @@ -48,7 +58,8 @@ export async function loadCliConfig(options: Options) { ? async name => process.env[name] || 'x' : undefined, configRoot: paths.targetRoot, - configPaths, + configPaths: [], + configTargets: configTargets, }); // printing to stderr to not clobber stdout in case the cli command diff --git a/packages/cli/src/lib/urls.test.ts b/packages/cli/src/lib/urls.test.ts new file mode 100644 index 0000000000..c2a67fb849 --- /dev/null +++ b/packages/cli/src/lib/urls.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isValidUrl } from './urls'; + +describe('isValidUrl', () => { + it('should return true for url', () => { + const validUrl = isValidUrl('http://some.valid.url'); + expect(validUrl).toBe(true); + }); + + it('should return false for absolute path', () => { + const validUrl = isValidUrl('/some/absolute/path'); + expect(validUrl).toBe(false); + }); + + it('should return false for relative path', () => { + const validUrl = isValidUrl('../some/relative/path'); + expect(validUrl).toBe(false); + }); +}); diff --git a/packages/cli/src/lib/urls.ts b/packages/cli/src/lib/urls.ts new file mode 100644 index 0000000000..848cea25d9 --- /dev/null +++ b/packages/cli/src/lib/urls.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isValidUrl(url: string): boolean { + try { + // eslint-disable-next-line no-new + new URL(url); + return true; + } catch { + return false; + } +} diff --git a/packages/config-loader/api-report.md b/packages/config-loader/api-report.md index 19b5ee9e27..6f568df942 100644 --- a/packages/config-loader/api-report.md +++ b/packages/config-loader/api-report.md @@ -23,6 +23,17 @@ export type ConfigSchemaProcessingOptions = { withFilteredKeys?: boolean; }; +// Warning: (ae-missing-release-tag) "ConfigTarget" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type ConfigTarget = + | { + path: string; + } + | { + url: string; + }; + // @public export type ConfigVisibility = 'frontend' | 'backend' | 'secret'; @@ -33,12 +44,26 @@ export function loadConfig(options: LoadConfigOptions): Promise; export type LoadConfigOptions = { configRoot: string; configPaths: string[]; + configTargets: ConfigTarget[]; env?: string; experimentalEnvFunc?: (name: string) => Promise; - watch?: { - onChange: (configs: AppConfig[]) => void; - stopSignal?: Promise; - }; + remote?: LoadConfigOptionsRemote; + watch?: LoadConfigOptionsWatch; +}; + +// Warning: (ae-missing-release-tag) "LoadConfigOptionsRemote" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type LoadConfigOptionsRemote = { + reloadIntervalSeconds: number; +}; + +// Warning: (ae-missing-release-tag) "LoadConfigOptionsWatch" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type LoadConfigOptionsWatch = { + onChange: (configs: AppConfig[]) => void; + stopSignal?: Promise; }; // @public @@ -71,4 +96,8 @@ export type TransformFunc = ( visibility: ConfigVisibility; }, ) => T | undefined; + +// Warnings were encountered during analysis: +// +// src/loader.d.ts:33:5 - (ae-unresolved-link) The @link reference could not be resolved: The package "@backstage/config-loader" does not have an export "configTargets" ``` diff --git a/packages/config-loader/package.json b/packages/config-loader/package.json index 604caa4008..910dd87108 100644 --- a/packages/config-loader/package.json +++ b/packages/config-loader/package.json @@ -43,7 +43,8 @@ "json-schema-traverse": "^1.0.0", "typescript-json-schema": "^0.51.0", "yaml": "^1.9.2", - "yup": "^0.32.9" + "yup": "^0.32.9", + "node-fetch": "2.6.5" }, "devDependencies": { "@types/jest": "^26.0.7", @@ -51,7 +52,8 @@ "@types/mock-fs": "^4.10.0", "@types/node": "^14.14.32", "@types/yup": "^0.29.13", - "mock-fs": "^5.1.0" + "mock-fs": "^5.1.0", + "msw": "^0.29.0" }, "files": [ "dist" diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index 888c68ef70..97f0d301a3 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -29,4 +29,9 @@ export type { TransformFunc, } from './lib'; export { loadConfig } from './loader'; -export type { LoadConfigOptions } from './loader'; +export type { + ConfigTarget, + LoadConfigOptions, + LoadConfigOptionsWatch, + LoadConfigOptionsRemote, +} from './loader'; diff --git a/packages/config-loader/src/lib/index.ts b/packages/config-loader/src/lib/index.ts index 32a0191cae..ca88b771ba 100644 --- a/packages/config-loader/src/lib/index.ts +++ b/packages/config-loader/src/lib/index.ts @@ -17,3 +17,4 @@ export { readEnvConfig } from './env'; export * from './transform'; export * from './schema'; +export { isValidUrl } from './urls'; diff --git a/packages/config-loader/src/lib/urls.test.ts b/packages/config-loader/src/lib/urls.test.ts new file mode 100644 index 0000000000..c2a67fb849 --- /dev/null +++ b/packages/config-loader/src/lib/urls.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isValidUrl } from './urls'; + +describe('isValidUrl', () => { + it('should return true for url', () => { + const validUrl = isValidUrl('http://some.valid.url'); + expect(validUrl).toBe(true); + }); + + it('should return false for absolute path', () => { + const validUrl = isValidUrl('/some/absolute/path'); + expect(validUrl).toBe(false); + }); + + it('should return false for relative path', () => { + const validUrl = isValidUrl('../some/relative/path'); + expect(validUrl).toBe(false); + }); +}); diff --git a/packages/config-loader/src/lib/urls.ts b/packages/config-loader/src/lib/urls.ts new file mode 100644 index 0000000000..848cea25d9 --- /dev/null +++ b/packages/config-loader/src/lib/urls.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function isValidUrl(url: string): boolean { + try { + // eslint-disable-next-line no-new + new URL(url); + return true; + } catch { + return false; + } +} diff --git a/packages/config-loader/src/loader.test.ts b/packages/config-loader/src/loader.test.ts index 3f51e76936..804254afa0 100644 --- a/packages/config-loader/src/loader.test.ts +++ b/packages/config-loader/src/loader.test.ts @@ -18,8 +18,43 @@ import { AppConfig } from '@backstage/config'; import { loadConfig } from './loader'; import mockFs from 'mock-fs'; import fs from 'fs-extra'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; describe('loadConfig', () => { + const server = setupServer(); + const initialLoaderHandler = rest.get( + `https://some.domain.io/app-config.yaml`, + (_req, res, ctx) => { + return res( + ctx.body( + `app: + title: Remote Example App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + ), + ); + }, + ); + + const reloadHandler = rest.get( + `https://some.domain.io/app-config.yaml`, + (_req, res, ctx) => { + return res( + ctx.body( + `app: + title: NEW ReMOTe ExaMPLe App + sessionKey: 'abc123' + escaped: \$\${Escaped} + `, + ), + ); + }, + ); + + beforeAll(() => server.listen()); + beforeEach(() => { process.env.MY_SECRET = 'is-secret'; process.env.SUBSTITUTE_ME = 'substituted'; @@ -32,6 +67,13 @@ describe('loadConfig', () => { $file: secrets/session-key.txt escaped: \$\${Escaped} `, + '/root/app-config2.yaml': ` + app: + title: Example App 2 + sessionKey: + $file: secrets/session-key.txt + escaped: \$\${Escaped} + `, '/root/app-config.development.yaml': ` app: sessionKey: development-key @@ -67,13 +109,17 @@ describe('loadConfig', () => { afterEach(() => { mockFs.restore(); + server.resetHandlers(); }); + afterAll(() => server.close()); + it('load config from default path', async () => { await expect( loadConfig({ configRoot: '/root', configPaths: [], + configTargets: [], env: 'production', }), ).resolves.toEqual([ @@ -90,11 +136,73 @@ describe('loadConfig', () => { ]); }); - it('loads config with secrets', async () => { + it('load config from remote path', async () => { + server.use(initialLoaderHandler); + + const configUrl = 'https://some.domain.io/app-config.yaml'; + + await expect( + loadConfig({ + configRoot: '/root', + configPaths: [], + configTargets: [{ url: configUrl }], + env: 'production', + remote: { + reloadIntervalSeconds: 30, + }, + }), + ).resolves.toEqual([ + { + context: configUrl, + data: { + app: { + title: 'Remote Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + }); + + it('loads config with secrets from two different files', async () => { + await expect( + loadConfig({ + configRoot: '/root', + configPaths: ['/root/app-config2.yaml'], + configTargets: [{ path: '/root/app-config.yaml' }], + env: 'production', + }), + ).resolves.toEqual([ + { + context: 'app-config.yaml', + data: { + app: { + title: 'Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + { + context: 'app-config2.yaml', + data: { + app: { + title: 'Example App 2', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + }); + + it('loads config with secrets from single file', async () => { await expect( loadConfig({ configRoot: '/root', configPaths: ['/root/app-config.yaml'], + configTargets: [{ path: '/root/app-config.yaml' }], env: 'production', }), ).resolves.toEqual([ @@ -115,9 +223,10 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: [ - '/root/app-config.yaml', - '/root/app-config.development.yaml', + configPaths: [], + configTargets: [ + { path: '/root/app-config.yaml' }, + { path: '/root/app-config.development.yaml' }, ], env: 'development', }), @@ -155,7 +264,8 @@ describe('loadConfig', () => { await expect( loadConfig({ configRoot: '/root', - configPaths: ['/root/app-config.substitute.yaml'], + configPaths: [], + configTargets: [{ path: '/root/app-config.substitute.yaml' }], env: 'development', }), ).resolves.toEqual([ @@ -181,6 +291,7 @@ describe('loadConfig', () => { loadConfig({ configRoot: '/root', configPaths: [], + configTargets: [], watch: { onChange: onChange.resolve, stopSignal: stopSignal.promise, @@ -218,12 +329,64 @@ describe('loadConfig', () => { stopSignal.resolve(); }); + it('watches remote config urls', async () => { + server.use(initialLoaderHandler); + + const onChange = defer(); + const stopSignal = defer(); + + const configUrl = 'https://some.domain.io/app-config.yaml'; + await expect( + loadConfig({ + configRoot: '/root', + configPaths: [], + configTargets: [{ url: configUrl }], + watch: { + onChange: onChange.resolve, + stopSignal: stopSignal.promise, + }, + remote: { + reloadIntervalSeconds: 1, + }, + }), + ).resolves.toEqual([ + { + context: configUrl, + data: { + app: { + title: 'Remote Example App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + + server.use(reloadHandler); + + await expect(onChange.promise).resolves.toEqual([ + { + context: configUrl, + data: { + app: { + title: 'NEW ReMOTe ExaMPLe App', + sessionKey: 'abc123', + escaped: '${Escaped}', + }, + }, + }, + ]); + + stopSignal.resolve(); + }); + it('stops watching config files', async () => { const stopSignal = defer(); await loadConfig({ configRoot: '/root', configPaths: [], + configTargets: [], watch: { onChange: () => { expect('not').toBe('called'); diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 6803d61786..4a5660f46c 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -17,15 +17,38 @@ import fs from 'fs-extra'; import yaml from 'yaml'; import chokidar from 'chokidar'; -import { resolve as resolvePath, dirname, isAbsolute, basename } from 'path'; +import { basename, dirname, isAbsolute, resolve as resolvePath } from 'path'; import { AppConfig } from '@backstage/config'; import { ForwardedError } from '@backstage/errors'; import { applyConfigTransforms, - readEnvConfig, createIncludeTransform, createSubstitutionTransform, + isValidUrl, + readEnvConfig, } from './lib'; +import fetch from 'node-fetch'; + +export type ConfigTarget = { path: string } | { url: string }; + +export type LoadConfigOptionsWatch = { + /** + * A listener that is called when a config file is changed. + */ + onChange: (configs: AppConfig[]) => void; + + /** + * An optional signal that stops the watcher once the promise resolves. + */ + stopSignal?: Promise; +}; + +export type LoadConfigOptionsRemote = { + /** + * An optional remote config reloading period, in seconds + */ + reloadIntervalSeconds: number; +}; /** * Options that control the loading of configuration files in the backend. @@ -36,9 +59,14 @@ export type LoadConfigOptions = { // The root directory of the config loading context. Used to find default configs. configRoot: string; - // Absolute paths to load config files from. Configs from earlier paths have lower priority. + /** Absolute paths to load config files from. Configs from earlier paths have lower priority. + * @deprecated Use {@link configTargets} instead. + */ configPaths: string[]; + // Paths to load config files from. Configs from earlier paths have lower priority. + configTargets: ConfigTarget[]; + /** @deprecated This option has been removed */ env?: string; @@ -49,20 +77,15 @@ export type LoadConfigOptions = { */ experimentalEnvFunc?: (name: string) => Promise; + /** + * An optional remote config + */ + remote?: LoadConfigOptionsRemote; + /** * An optional configuration that enables watching of config files. */ - watch?: { - /** - * A listener that is called when a config file is changed. - */ - onChange: (configs: AppConfig[]) => void; - - /** - * An optional signal that stops the watcher once the promise resolves. - */ - stopSignal?: Promise; - }; + watch?: LoadConfigOptionsWatch; }; /** @@ -73,12 +96,32 @@ export type LoadConfigOptions = { export async function loadConfig( options: LoadConfigOptions, ): Promise { - const { configRoot, experimentalEnvFunc: envFunc, watch } = options; - const configPaths = options.configPaths.slice(); + const { configRoot, experimentalEnvFunc: envFunc, watch, remote } = options; + + const configPaths: string[] = options.configTargets + .slice() + .filter((e): e is { path: string } => e.hasOwnProperty('path')) + .map(configTarget => configTarget.path); + + // Append deprecated configPaths to the absolute config paths received via configTargets. + options.configPaths.forEach(cp => { + if (!configPaths.includes(cp)) { + configPaths.push(cp); + } + }); + + const configUrls: string[] = options.configTargets + .slice() + .filter((e): e is { url: string } => e.hasOwnProperty('url')) + .map(configTarget => configTarget.url); + + if (remote === undefined && configUrls.length > 0) { + throw new Error(`Remote config detected but this feature is turned off`); + } // If no paths are provided, we default to reading // `app-config.yaml` and, if it exists, `app-config.local.yaml` - if (configPaths.length === 0) { + if (configPaths.length === 0 && configUrls.length === 0) { configPaths.push(resolvePath(configRoot, 'app-config.yaml')); const localConfig = resolvePath(configRoot, 'app-config.local.yaml'); @@ -114,22 +157,67 @@ export async function loadConfig( return configs; }; - let fileConfigs; + const loadRemoteConfigFiles = async () => { + const configs: AppConfig[] = []; + + const readConfigFromUrl = async (url: string) => { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Could not read config file at ${url}`); + } + + return await response.text(); + }; + + for (let i = 0; i < configUrls.length; i++) { + const configUrl = configUrls[i]; + if (!isValidUrl(configUrl)) { + throw new Error(`Config load path is not valid: '${configUrl}'`); + } + + const remoteConfigContent = await readConfigFromUrl(configUrl); + if (!remoteConfigContent) { + throw new Error(`Config is not valid`); + } + const configYaml = yaml.parse(remoteConfigContent); + const substitutionTransform = createSubstitutionTransform(env); + const data = await applyConfigTransforms(configRoot, configYaml, [ + substitutionTransform, + ]); + + configs.push({ data, context: configUrl }); + } + + return configs; + }; + + let fileConfigs: AppConfig[]; try { fileConfigs = await loadConfigFiles(); } catch (error) { throw new ForwardedError('Failed to read static configuration file', error); } + let remoteConfigs: AppConfig[] = []; + if (remote) { + try { + remoteConfigs = await loadRemoteConfigFiles(); + } catch (error) { + throw new ForwardedError( + `Failed to read remote configuration file`, + error, + ); + } + } + const envConfigs = await readEnvConfig(process.env); - // Set up config file watching if requested by the caller - if (watch) { - let currentSerializedConfig = JSON.stringify(fileConfigs); - + const watchConfigFile = (watchProp: LoadConfigOptionsWatch) => { const watcher = chokidar.watch(configPaths, { usePolling: process.env.NODE_ENV === 'test', }); + + let currentSerializedConfig = JSON.stringify(fileConfigs); watcher.on('change', async () => { try { const newConfigs = await loadConfigFiles(); @@ -140,18 +228,69 @@ export async function loadConfig( } currentSerializedConfig = newSerializedConfig; - watch.onChange([...newConfigs, ...envConfigs]); + watchProp.onChange([...remoteConfigs, ...newConfigs, ...envConfigs]); } catch (error) { console.error(`Failed to reload configuration files, ${error}`); } }); - if (watch.stopSignal) { - watch.stopSignal.then(() => { + if (watchProp.stopSignal) { + watchProp.stopSignal.then(() => { watcher.close(); }); } + }; + + const watchRemoteConfig = ( + watchProp: LoadConfigOptionsWatch, + remoteProp: LoadConfigOptionsRemote, + ) => { + const hasConfigChanged = async ( + oldRemoteConfigs: AppConfig[], + newRemoteConfigs: AppConfig[], + ) => { + return ( + JSON.stringify(oldRemoteConfigs) !== JSON.stringify(newRemoteConfigs) + ); + }; + + let handle: NodeJS.Timeout | undefined; + try { + handle = setInterval(async () => { + console.info(`Checking for config update`); + const newRemoteConfigs = await loadRemoteConfigFiles(); + if (await hasConfigChanged(remoteConfigs, newRemoteConfigs)) { + remoteConfigs = newRemoteConfigs; + console.info(`Remote config change, reloading config ...`); + watchProp.onChange([...remoteConfigs, ...fileConfigs, ...envConfigs]); + console.info(`Remote config reloaded`); + } + }, remoteProp.reloadIntervalSeconds * 1000); + } catch (error) { + console.error(`Failed to reload configuration files, ${error}`); + } + + if (watchProp.stopSignal) { + watchProp.stopSignal.then(() => { + if (handle !== undefined) { + console.info(`Stopping remote config watch`); + clearInterval(handle); + handle = undefined; + } + }); + } + }; + + // Set up config file watching if requested by the caller + if (watch) { + watchConfigFile(watch); } - return [...fileConfigs, ...envConfigs]; + if (watch && remote) { + watchRemoteConfig(watch, remote); + } + + return remote + ? [...remoteConfigs, ...fileConfigs, ...envConfigs] + : [...fileConfigs, ...envConfigs]; } diff --git a/yarn.lock b/yarn.lock index 3fec636e32..e88a6ca99e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14211,6 +14211,11 @@ fecha@^4.2.0: resolved "https://registry.npmjs.org/fecha/-/fecha-4.2.0.tgz#3ffb6395453e3f3efff850404f0a59b6747f5f41" integrity sha512-aN3pcx/DSmtyoovUudctc8+6Hl4T+hI9GBBHLjA76jdZl7+b1sgh5g4k+u/GL3dTy1/pnYzKp69FpJ0OicE3Wg== +fetch-blob@2.1.2: + version "2.1.2" + resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.2.tgz#a7805db1361bd44c1ef62bb57fb5fe8ea173ef3c" + integrity sha512-YKqtUDwqLyfyMnmbw8XD6Q8j9i/HggKtPEI+pZ1+8bvheBu78biSmNaXWusx1TauGqtUUGx/cBb1mKdq2rLYow== + fetch-readablestream@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/fetch-readablestream/-/fetch-readablestream-0.2.0.tgz#eaa6d1a76b12de2d4731a343393c6ccdcfe2c795" @@ -20703,6 +20708,13 @@ node-fetch@2.6.1, node-fetch@^2.3.0, node-fetch@^2.6.0, node-fetch@^2.6.1: resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== +node-fetch@2.6.5: + version "2.6.5" + resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz#42735537d7f080a7e5f78b6c549b7146be1742fd" + integrity sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ== + dependencies: + whatwg-url "^5.0.0" + node-forge@^0.10.0: version "0.10.0" resolved "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3" @@ -27091,6 +27103,11 @@ tr46@^2.0.2: dependencies: punycode "^2.1.1" +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o= + "traverse@>=0.3.0 <0.4": version "0.3.9" resolved "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz#717b8f220cc0bb7b44e40514c22b2e8bbc70d8b9" @@ -28340,6 +28357,16 @@ web-streams-polyfill@4.0.0-beta.1: resolved "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.1.tgz#3b19b9817374b7cee06d374ba7eeb3aeb80e8c95" integrity sha512-3ux37gEX670UUphBF9AMCq8XM6iQ8Ac6A+DSRRjDoRBm1ufCkaCDdNVbaqq60PsEkdNlLKrGtv/YBP4EJXqNtQ== +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE= + +webidl-conversions@^4.0.2: + version "4.0.2" + resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad" + integrity sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg== + webidl-conversions@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff" @@ -28555,6 +28582,32 @@ whatwg-mimetype@^2.3.0: resolved "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0= + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + +whatwg-url@^6.4.1: + version "6.5.0" + resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz#f2df02bff176fd65070df74ad5ccbb5a199965a8" + integrity sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ== + dependencies: + lodash.sortby "^4.7.0" + tr46 "^1.0.1" + webidl-conversions "^4.0.2" + +whatwg-url@^7.0.0: + version "7.1.0" + resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz#c2c492f1eca612988efd3d2266be1b9fc6170d06" + integrity sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg== + dependencies: + lodash.sortby "^4.7.0" + tr46 "^1.0.1" + webidl-conversions "^4.0.2" + whatwg-url@^8.0.0, whatwg-url@^8.4.0: version "8.4.0" resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.4.0.tgz#50fb9615b05469591d2b2bd6dfaed2942ed72837"